dexverse-cli 0.0.2__tar.gz → 0.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/PKG-INFO +10 -10
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/README.md +8 -8
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/__init__.py +1 -1
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/convert/hdf5_to_raw.py +10 -10
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/downloader/manager.py +9 -9
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/downloader/state.py +5 -5
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/manifest.py +25 -3
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/examples/download (7).json +8 -8
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/pyproject.toml +7 -1
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/set-creds.ps1 +3 -3
- dexverse_cli-0.0.2/tests/test_convert_task_instruction.py → dexverse_cli-0.0.4/tests/test_convert_additional_info.py +14 -16
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_http_backend.py +15 -15
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_manifest_aliases.py +53 -13
- dexverse_cli-0.0.2/648584.hdf5 +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/.dockerignore +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/.gitignore +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/Dockerfile +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/cli.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/client.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/config.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/convert/__init__.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/convert/image_video_encode.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/downloader/__init__.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/downloader/base.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/downloader/http_backend.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/downloader/python_s3_backend.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/downloader/s5cmd_backend.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/downloader/verify.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/mock/__init__.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/mock/mock_tasks.json +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/mock/server.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/docker_run.sh +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/docs/DexVerse_Downloader_S3_Final_/345/256/236/346/226/275/346/226/271/346/241/210_/345/216/237/350/267/257/345/276/204/350/220/275/347/233/230/347/211/210.md" +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/docs/PROMPT_copy_and_download_command.md +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/docs/Python_vs_Node_CLI/345/257/271/346/257/224/345/210/206/346/236/220.md" +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/docs/Python_vs_Node_/346/216/245/345/205/245C++/345/257/271/346/257/224/345/210/206/346/236/220.md" +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/docs//345/210/206/345/217/221/347/273/231/347/256/227/346/263/225/345/233/242/351/230/237_/346/225/231/347/250/213.md" +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/docs//346/223/215/344/275/234/346/214/207/345/215/227--/345/234/250Dexechain/344/270/255/350/277/233/350/241/214/346/250/241/344/273/277/345/255/246/344/271/240.md" +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/docs//346/234/215/345/212/241/345/231/250/344/275/277/347/224/250/346/211/213/345/206/214.md" +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/examples/local_demo.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/examples/manifest.json +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/examples/manifest_presigned.json +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/conftest.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_cli.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_http_resilience.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_manager_integration.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_manifest.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_mock_server.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_presigned_fixes.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_review_fixes.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_s5cmd_generation.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_state.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_verify.py +0 -0
- {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: dexverse-cli
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.4
|
|
4
4
|
Summary: DexVerse dataset downloader for S3-compatible robot training datasets
|
|
5
5
|
Author: DexVerse
|
|
6
6
|
License: Apache-2.0
|
|
@@ -86,7 +86,7 @@ Raw backend output is accepted directly:
|
|
|
86
86
|
"storage_path": "datasets/.../20260429_180359.hdf5",
|
|
87
87
|
"file_name": "20260429_180359.hdf5",
|
|
88
88
|
"file_size": 14994636,
|
|
89
|
-
"
|
|
89
|
+
"additional_info": {
|
|
90
90
|
"aaa": "111",
|
|
91
91
|
"aaaa": 222,
|
|
92
92
|
"ewrwrwe": 1.1111111111111112e+54
|
|
@@ -100,7 +100,7 @@ Result:
|
|
|
100
100
|
D:\DexVerse\20260429_180359.hdf5
|
|
101
101
|
```
|
|
102
102
|
|
|
103
|
-
`
|
|
103
|
+
`additional_info` is optional and must be a JSON object. If multiple manifest
|
|
104
104
|
items resolve to the same local path (for example, repeated `file_name`
|
|
105
105
|
values), the first item wins and later duplicates are ignored.
|
|
106
106
|
|
|
@@ -133,8 +133,8 @@ dexverse-dl repair --manifest PATH --out /data/dexverse [--backend http|python|
|
|
|
133
133
|
### 还原 HDF5 → 可训练数据
|
|
134
134
|
|
|
135
135
|
下载完成后,把落盘的 `.hdf5` 还原为原始录制结构(逐帧 JPEG + `pose_record_*.json`
|
|
136
|
-
+ `metadata.jsonl`)。如果对应 manifest 带有 `
|
|
137
|
-
`
|
|
136
|
+
+ `metadata.jsonl`)。如果对应 manifest 带有 `additional_info`,还会自动生成
|
|
137
|
+
`additional_info.json`:
|
|
138
138
|
|
|
139
139
|
```bash
|
|
140
140
|
# 单文件
|
|
@@ -153,11 +153,11 @@ dexverse-dl convert --input-dir /data/dexverse -o /data/dexverse/restored
|
|
|
153
153
|
hand/right/000000.jpg ...
|
|
154
154
|
pose_record_20260429_180359.json
|
|
155
155
|
metadata.jsonl
|
|
156
|
-
|
|
156
|
+
additional_info.json
|
|
157
157
|
```
|
|
158
158
|
|
|
159
|
-
`
|
|
160
|
-
下载器会把转换所需的映射保存在 `.dexverse/
|
|
159
|
+
`additional_info.json` 的内容就是 manifest 中的对象本身,不增加外层包装。
|
|
160
|
+
下载器会把转换所需的映射保存在 `.dexverse/additional-info/`,无需为
|
|
161
161
|
`convert` 增加额外参数。独立来源且没有该隐藏元数据的 HDF5 仍按原行为还原。
|
|
162
162
|
|
|
163
163
|
该命令依赖 `h5py` / `numpy` / `av` / `pillow`,已列入核心依赖,安装时随包装上。
|
|
@@ -180,7 +180,7 @@ State files live under `<out>/.dexverse/` and never mix into the dataset tree:
|
|
|
180
180
|
|
|
181
181
|
```text
|
|
182
182
|
/data/dexverse/
|
|
183
|
-
.dexverse/
|
|
183
|
+
.dexverse/additional-info/
|
|
184
184
|
20260429_180359.hdf5.json
|
|
185
185
|
.dexverse/tasks/<taskId>/
|
|
186
186
|
manifest.json
|
|
@@ -56,7 +56,7 @@ Raw backend output is accepted directly:
|
|
|
56
56
|
"storage_path": "datasets/.../20260429_180359.hdf5",
|
|
57
57
|
"file_name": "20260429_180359.hdf5",
|
|
58
58
|
"file_size": 14994636,
|
|
59
|
-
"
|
|
59
|
+
"additional_info": {
|
|
60
60
|
"aaa": "111",
|
|
61
61
|
"aaaa": 222,
|
|
62
62
|
"ewrwrwe": 1.1111111111111112e+54
|
|
@@ -70,7 +70,7 @@ Result:
|
|
|
70
70
|
D:\DexVerse\20260429_180359.hdf5
|
|
71
71
|
```
|
|
72
72
|
|
|
73
|
-
`
|
|
73
|
+
`additional_info` is optional and must be a JSON object. If multiple manifest
|
|
74
74
|
items resolve to the same local path (for example, repeated `file_name`
|
|
75
75
|
values), the first item wins and later duplicates are ignored.
|
|
76
76
|
|
|
@@ -103,8 +103,8 @@ dexverse-dl repair --manifest PATH --out /data/dexverse [--backend http|python|
|
|
|
103
103
|
### 还原 HDF5 → 可训练数据
|
|
104
104
|
|
|
105
105
|
下载完成后,把落盘的 `.hdf5` 还原为原始录制结构(逐帧 JPEG + `pose_record_*.json`
|
|
106
|
-
+ `metadata.jsonl`)。如果对应 manifest 带有 `
|
|
107
|
-
`
|
|
106
|
+
+ `metadata.jsonl`)。如果对应 manifest 带有 `additional_info`,还会自动生成
|
|
107
|
+
`additional_info.json`:
|
|
108
108
|
|
|
109
109
|
```bash
|
|
110
110
|
# 单文件
|
|
@@ -123,11 +123,11 @@ dexverse-dl convert --input-dir /data/dexverse -o /data/dexverse/restored
|
|
|
123
123
|
hand/right/000000.jpg ...
|
|
124
124
|
pose_record_20260429_180359.json
|
|
125
125
|
metadata.jsonl
|
|
126
|
-
|
|
126
|
+
additional_info.json
|
|
127
127
|
```
|
|
128
128
|
|
|
129
|
-
`
|
|
130
|
-
下载器会把转换所需的映射保存在 `.dexverse/
|
|
129
|
+
`additional_info.json` 的内容就是 manifest 中的对象本身,不增加外层包装。
|
|
130
|
+
下载器会把转换所需的映射保存在 `.dexverse/additional-info/`,无需为
|
|
131
131
|
`convert` 增加额外参数。独立来源且没有该隐藏元数据的 HDF5 仍按原行为还原。
|
|
132
132
|
|
|
133
133
|
该命令依赖 `h5py` / `numpy` / `av` / `pillow`,已列入核心依赖,安装时随包装上。
|
|
@@ -150,7 +150,7 @@ State files live under `<out>/.dexverse/` and never mix into the dataset tree:
|
|
|
150
150
|
|
|
151
151
|
```text
|
|
152
152
|
/data/dexverse/
|
|
153
|
-
.dexverse/
|
|
153
|
+
.dexverse/additional-info/
|
|
154
154
|
20260429_180359.hdf5.json
|
|
155
155
|
.dexverse/tasks/<taskId>/
|
|
156
156
|
manifest.json
|
|
@@ -35,7 +35,7 @@ from ..downloader import state as download_state
|
|
|
35
35
|
from .image_video_encode import decode_mp4_bytes_to_rgb_images
|
|
36
36
|
|
|
37
37
|
|
|
38
|
-
|
|
38
|
+
ADDITIONAL_INFO_FILENAME = "additional_info.json"
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
def _find_download_root(hdf5_path: Path) -> Optional[Path]:
|
|
@@ -47,26 +47,26 @@ def _find_download_root(hdf5_path: Path) -> Optional[Path]:
|
|
|
47
47
|
return None
|
|
48
48
|
|
|
49
49
|
|
|
50
|
-
def
|
|
50
|
+
def _sync_restored_additional_info(
|
|
51
51
|
hdf5_path: Path,
|
|
52
52
|
output_dir: Path,
|
|
53
53
|
) -> None:
|
|
54
|
-
"""Copy a downloaded file's hidden
|
|
54
|
+
"""Copy a downloaded file's hidden additional info into its restored session."""
|
|
55
55
|
download_root = _find_download_root(hdf5_path)
|
|
56
56
|
if download_root is None:
|
|
57
57
|
return
|
|
58
58
|
|
|
59
59
|
resolved_hdf5 = Path(hdf5_path).expanduser().resolve()
|
|
60
60
|
relpath = resolved_hdf5.relative_to(download_root).as_posix()
|
|
61
|
-
source = download_state.
|
|
62
|
-
destination = Path(output_dir) /
|
|
61
|
+
source = download_state.additional_info_path(download_root, relpath)
|
|
62
|
+
destination = Path(output_dir) / ADDITIONAL_INFO_FILENAME
|
|
63
63
|
|
|
64
64
|
if source.is_file():
|
|
65
|
-
|
|
66
|
-
if not isinstance(
|
|
67
|
-
raise ValueError(f"
|
|
65
|
+
additional_info = json.loads(source.read_text(encoding="utf-8"))
|
|
66
|
+
if not isinstance(additional_info, dict):
|
|
67
|
+
raise ValueError(f"additional info must be a JSON object: {source}")
|
|
68
68
|
destination.write_text(
|
|
69
|
-
json.dumps(
|
|
69
|
+
json.dumps(additional_info, ensure_ascii=False, indent=2) + "\n",
|
|
70
70
|
encoding="utf-8",
|
|
71
71
|
)
|
|
72
72
|
else:
|
|
@@ -249,7 +249,7 @@ class HDF5ToRawConverter:
|
|
|
249
249
|
img = Image.open(_io.BytesIO(jpg_bytes)).convert("RGB")
|
|
250
250
|
img.save(cam_dir / f"{i:06d}.jpg", "JPEG", quality=95)
|
|
251
251
|
|
|
252
|
-
|
|
252
|
+
_sync_restored_additional_info(hdf5_path, output_dir)
|
|
253
253
|
return frames_count
|
|
254
254
|
|
|
255
255
|
def _extract_frames(self, h5_file: h5py.File) -> List[AlignedFrame]:
|
|
@@ -43,26 +43,26 @@ def _persist_manifest_copy(out: Path, task_id: str, manifest: Manifest) -> None:
|
|
|
43
43
|
pass
|
|
44
44
|
|
|
45
45
|
|
|
46
|
-
def
|
|
46
|
+
def _sync_additional_info(
|
|
47
47
|
out: Path,
|
|
48
48
|
manifest: Manifest,
|
|
49
49
|
task_state: st.TaskState,
|
|
50
50
|
) -> None:
|
|
51
|
-
"""Synchronize per-file
|
|
51
|
+
"""Synchronize per-file additional info after download/verification."""
|
|
52
52
|
successful_statuses = {st.DOWNLOADED, st.SKIPPED, st.VERIFIED}
|
|
53
53
|
for file_spec in manifest.files:
|
|
54
54
|
key = file_spec.effective_key()
|
|
55
55
|
file_state = task_state.files.get(key)
|
|
56
|
-
|
|
56
|
+
info_path = st.additional_info_path(out, file_spec.relpath())
|
|
57
57
|
if (
|
|
58
58
|
file_state is not None
|
|
59
59
|
and file_state.status in successful_statuses
|
|
60
|
-
and file_spec.
|
|
60
|
+
and file_spec.additionalInfo is not None
|
|
61
61
|
):
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
info_path.parent.mkdir(parents=True, exist_ok=True)
|
|
63
|
+
info_path.write_text(
|
|
64
64
|
json.dumps(
|
|
65
|
-
file_spec.
|
|
65
|
+
file_spec.additionalInfo,
|
|
66
66
|
ensure_ascii=False,
|
|
67
67
|
indent=2,
|
|
68
68
|
)
|
|
@@ -70,7 +70,7 @@ def _sync_task_instructions(
|
|
|
70
70
|
encoding="utf-8",
|
|
71
71
|
)
|
|
72
72
|
else:
|
|
73
|
-
|
|
73
|
+
info_path.unlink(missing_ok=True)
|
|
74
74
|
|
|
75
75
|
|
|
76
76
|
def get_backend(name: str, **kwargs) -> Backend:
|
|
@@ -280,7 +280,7 @@ class DownloadManager:
|
|
|
280
280
|
verified += 1
|
|
281
281
|
task_state.mark(key, st.VERIFIED)
|
|
282
282
|
|
|
283
|
-
|
|
283
|
+
_sync_additional_info(self.out, manifest, task_state)
|
|
284
284
|
task_state.finish()
|
|
285
285
|
task_state.save()
|
|
286
286
|
|
|
@@ -17,7 +17,7 @@ from ..manifest import Manifest, resolve_dest
|
|
|
17
17
|
|
|
18
18
|
STATE_DIRNAME = ".dexverse"
|
|
19
19
|
TASKS_DIRNAME = "tasks"
|
|
20
|
-
|
|
20
|
+
ADDITIONAL_INFO_DIRNAME = "additional-info"
|
|
21
21
|
|
|
22
22
|
# file statuses
|
|
23
23
|
PENDING = "pending"
|
|
@@ -36,13 +36,13 @@ def task_dir(out: Path, task_id: str) -> Path:
|
|
|
36
36
|
return dexverse_root(out) / TASKS_DIRNAME / task_id
|
|
37
37
|
|
|
38
38
|
|
|
39
|
-
def
|
|
40
|
-
return dexverse_root(out) /
|
|
39
|
+
def additional_info_dir(out: Path) -> Path:
|
|
40
|
+
return dexverse_root(out) / ADDITIONAL_INFO_DIRNAME
|
|
41
41
|
|
|
42
42
|
|
|
43
|
-
def
|
|
43
|
+
def additional_info_path(out: Path, relpath: str) -> Path:
|
|
44
44
|
"""Hidden metadata path for one downloaded dataset file."""
|
|
45
|
-
return resolve_dest(
|
|
45
|
+
return resolve_dest(additional_info_dir(out), f"{relpath}.json")
|
|
46
46
|
|
|
47
47
|
|
|
48
48
|
def state_file_path(out: Path, task_id: str) -> Path:
|
|
@@ -160,11 +160,33 @@ class FileSpec(BaseModel):
|
|
|
160
160
|
# Consumed only by the ``http`` backend; ignored by s5cmd/python.
|
|
161
161
|
presignedUrl: Optional[str] = Field(
|
|
162
162
|
None, validation_alias=AliasChoices("presignedUrl", "download_url", "url", "downloadUrl"))
|
|
163
|
-
|
|
163
|
+
additionalInfo: Optional[Dict[str, Any]] = Field(
|
|
164
164
|
None,
|
|
165
|
-
validation_alias=AliasChoices("
|
|
165
|
+
validation_alias=AliasChoices("additionalInfo", "additional_info"),
|
|
166
166
|
)
|
|
167
167
|
|
|
168
|
+
@model_validator(mode="before")
|
|
169
|
+
@classmethod
|
|
170
|
+
def validate_metadata_fields(cls, data: Any) -> Any:
|
|
171
|
+
"""Reject ambiguous aliases and removed names before extras are kept."""
|
|
172
|
+
if isinstance(data, dict):
|
|
173
|
+
if "additional_info" in data and "additionalInfo" in data:
|
|
174
|
+
raise ValueError(
|
|
175
|
+
"fields 'additional_info' and 'additionalInfo' cannot be "
|
|
176
|
+
"used together; provide exactly one"
|
|
177
|
+
)
|
|
178
|
+
removed = [
|
|
179
|
+
name
|
|
180
|
+
for name in ("task_instruction", "taskInstruction")
|
|
181
|
+
if name in data
|
|
182
|
+
]
|
|
183
|
+
if removed:
|
|
184
|
+
names = ", ".join(repr(name) for name in removed)
|
|
185
|
+
raise ValueError(
|
|
186
|
+
f"removed field(s) {names}; use 'additional_info' instead"
|
|
187
|
+
)
|
|
188
|
+
return data
|
|
189
|
+
|
|
168
190
|
def relpath(self) -> str:
|
|
169
191
|
"""The local landing path relative to out (POSIX, normalized)."""
|
|
170
192
|
return resolve_relpath(self.relativePath, self.originalPath, self.objectKey)
|
|
@@ -217,7 +239,7 @@ class Manifest(BaseModel):
|
|
|
217
239
|
|
|
218
240
|
Backend-native manifests commonly provide only ``file_name`` as the
|
|
219
241
|
landing path. Keeping the first occurrence makes duplicate names
|
|
220
|
-
deterministic and ensures the downloaded file and its
|
|
242
|
+
deterministic and ensures the downloaded file and its additional info
|
|
221
243
|
always come from the same manifest entry.
|
|
222
244
|
"""
|
|
223
245
|
seen = set()
|
|
@@ -3,14 +3,14 @@
|
|
|
3
3
|
{
|
|
4
4
|
"download_url": "https://192.168.3.55:8443/dexverse/datasets/2606261645500010275566473183232/2607240417070338897362032267265/20260710_181607.hdf5?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260724%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20260724T070457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=333996e3875a8a7cdc65f26edd12de30298a99f68a4e24ec10a7a187d65099c1",
|
|
5
5
|
"file_name": "20260710_181607.hdf5",
|
|
6
|
-
"
|
|
6
|
+
"additional_info": {
|
|
7
7
|
"aaa33333": "111"
|
|
8
8
|
}
|
|
9
9
|
},
|
|
10
10
|
{
|
|
11
11
|
"download_url": "https://192.168.3.55:8443/dexverse/datasets/2606261645500010275566473183232/2607230920230338611294116515841/20260630_152359.hdf5?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260724%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20260724T070457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=0207422c867030015aa31c66aac264b70d9f67d508b2c08e9278b5dc85074a23",
|
|
12
12
|
"file_name": "20260630_152359.hdf5",
|
|
13
|
-
"
|
|
13
|
+
"additional_info": {
|
|
14
14
|
"aaa": "111",
|
|
15
15
|
"aaaa": 222,
|
|
16
16
|
"ewrwrwe": 1.1111111111111112e+54
|
|
@@ -19,37 +19,37 @@
|
|
|
19
19
|
{
|
|
20
20
|
"download_url": "https://192.168.3.55:8443/dexverse/datasets/2606261645500010275566473183232/2607230304150338516637072232449/20260617_122549.hdf5?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260724%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20260724T070457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=8b0228bd8e53e023081295c2e4811625d03b0f4998f3631c5d868c85c6c19cb3",
|
|
21
21
|
"file_name": "20260617_122549.hdf5",
|
|
22
|
-
"
|
|
22
|
+
"additional_info": {
|
|
23
23
|
"aaa": "5555"
|
|
24
24
|
}
|
|
25
25
|
},
|
|
26
26
|
{
|
|
27
27
|
"download_url": "https://192.168.3.55:8443/dexverse/datasets/2606261645500010275566473183232/2607230255400338514476854677505/20260630_152119.hdf5?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260724%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20260724T070457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=5ec492d0d867d9b4e5c714e2b530da8949851c58585504b5e6d6843134073078",
|
|
28
28
|
"file_name": "20260630_152119.hdf5",
|
|
29
|
-
"
|
|
29
|
+
"additional_info": {
|
|
30
30
|
"aaa": "111"
|
|
31
31
|
}
|
|
32
32
|
},
|
|
33
33
|
{
|
|
34
34
|
"download_url": "https://192.168.3.55:8443/dexverse/datasets/2606261645500010275566473183232/2607230251340338513446729420801/20260710_181607.hdf5?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260724%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20260724T070457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=21703ed3690b5ea9d008e6ad541e6f1682f75a811cc4a7c0d47ff7e952ce4e65",
|
|
35
35
|
"file_name": "20260710_181607.hdf5",
|
|
36
|
-
"
|
|
36
|
+
"additional_info": {
|
|
37
37
|
"aaa": "111"
|
|
38
38
|
}
|
|
39
39
|
},
|
|
40
40
|
{
|
|
41
41
|
"download_url": "https://192.168.3.55:8443/dexverse/datasets/2606261645500010275566473183232/2607230250150338513114959974401/20260630_152428.hdf5?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260724%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20260724T070457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=608bb92fb07b871567f33d16729c33fb6a97c9d1960644da9b472e83009550b9",
|
|
42
42
|
"file_name": "20260630_152428.hdf5",
|
|
43
|
-
"
|
|
43
|
+
"additional_info": {
|
|
44
44
|
"aaa": "111"
|
|
45
45
|
}
|
|
46
46
|
},
|
|
47
47
|
{
|
|
48
48
|
"download_url": "https://192.168.3.55:8443/dexverse/datasets/2606261645500010275566473183232/2607221248330338301292864606209/20260408_212630.hdf5?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260724%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20260724T070457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=33e59a77325ad81b3c1f5294126ede617366cdbf3416d7ab79678fda75d65747",
|
|
49
49
|
"file_name": "20260408_212630.hdf5",
|
|
50
|
-
"
|
|
50
|
+
"additional_info": {
|
|
51
51
|
"aaa": "111"
|
|
52
52
|
}
|
|
53
53
|
}
|
|
54
54
|
]
|
|
55
|
-
}
|
|
55
|
+
}
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "dexverse-cli"
|
|
7
|
-
version = "0.0.
|
|
7
|
+
version = "0.0.4"
|
|
8
8
|
description = "DexVerse dataset downloader for S3-compatible robot training datasets"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -48,6 +48,12 @@ dexverse-mock-server = "dexverse.mock.server:main"
|
|
|
48
48
|
[tool.hatch.build.targets.wheel]
|
|
49
49
|
packages = ["dexverse"]
|
|
50
50
|
|
|
51
|
+
[tool.hatch.build]
|
|
52
|
+
exclude = [
|
|
53
|
+
"/dexverse/dexe_cloud_upload/**",
|
|
54
|
+
"/648584.hdf5",
|
|
55
|
+
]
|
|
56
|
+
|
|
51
57
|
[tool.pytest.ini_options]
|
|
52
58
|
testpaths = ["tests"]
|
|
53
59
|
addopts = "-ra -q"
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
$env:AWS_ACCESS_KEY_ID="AKIA...真实access_key"
|
|
2
|
-
$env:AWS_SECRET_ACCESS_KEY="真实secret_key"
|
|
3
|
-
$env:AWS_REGION="us-east-1"
|
|
1
|
+
$env:AWS_ACCESS_KEY_ID="AKIA...真实access_key"
|
|
2
|
+
$env:AWS_SECRET_ACCESS_KEY="真实secret_key"
|
|
3
|
+
$env:AWS_REGION="us-east-1"
|
|
@@ -20,22 +20,22 @@ def _write_minimal_hdf5(path: Path) -> None:
|
|
|
20
20
|
joints.attrs["columns"] = json.dumps(["joint"])
|
|
21
21
|
|
|
22
22
|
|
|
23
|
-
def
|
|
24
|
-
path = st.
|
|
23
|
+
def _write_hidden_additional_info(root: Path, relpath: str, info: dict) -> Path:
|
|
24
|
+
path = st.additional_info_path(root, relpath)
|
|
25
25
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
26
26
|
path.write_text(
|
|
27
|
-
json.dumps(
|
|
27
|
+
json.dumps(info, ensure_ascii=False, indent=2) + "\n",
|
|
28
28
|
encoding="utf-8",
|
|
29
29
|
)
|
|
30
30
|
return path
|
|
31
31
|
|
|
32
32
|
|
|
33
|
-
def
|
|
33
|
+
def test_convert_single_file_writes_and_updates_raw_additional_info(tmp_path):
|
|
34
34
|
download_root = tmp_path / "download"
|
|
35
35
|
source = download_root / "session.hdf5"
|
|
36
36
|
restored = tmp_path / "restored" / "session"
|
|
37
37
|
_write_minimal_hdf5(source)
|
|
38
|
-
|
|
38
|
+
info_path = _write_hidden_additional_info(
|
|
39
39
|
download_root,
|
|
40
40
|
"session.hdf5",
|
|
41
41
|
{"aaa": "中文", "number": 1.1111111111111112e+54},
|
|
@@ -44,36 +44,34 @@ def test_convert_single_file_writes_and_updates_raw_task_instruction(tmp_path):
|
|
|
44
44
|
frames = convert_hdf5(source, restored)
|
|
45
45
|
|
|
46
46
|
assert frames == 1
|
|
47
|
-
output_path = restored / "
|
|
47
|
+
output_path = restored / "additional_info.json"
|
|
48
48
|
assert json.loads(output_path.read_text(encoding="utf-8")) == {
|
|
49
49
|
"aaa": "中文",
|
|
50
50
|
"number": 1.1111111111111112e+54,
|
|
51
51
|
}
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
instruction_path.write_text('{"updated": 2}\n', encoding="utf-8")
|
|
52
|
+
info_path.write_text('{"updated": 2}\n', encoding="utf-8")
|
|
55
53
|
convert_hdf5(source, restored)
|
|
56
54
|
assert json.loads(output_path.read_text(encoding="utf-8")) == {"updated": 2}
|
|
57
55
|
|
|
58
|
-
|
|
56
|
+
info_path.unlink()
|
|
59
57
|
convert_hdf5(source, restored)
|
|
60
58
|
assert not output_path.exists()
|
|
61
59
|
|
|
62
60
|
|
|
63
|
-
def
|
|
61
|
+
def test_convert_batch_preserves_paths_for_additional_info(tmp_path):
|
|
64
62
|
download_root = tmp_path / "download"
|
|
65
63
|
source = download_root / "nested" / "batch.hdf5"
|
|
66
64
|
restored_root = tmp_path / "restored"
|
|
67
65
|
_write_minimal_hdf5(source)
|
|
68
|
-
|
|
69
|
-
|
|
66
|
+
info = {"task": "batch"}
|
|
67
|
+
_write_hidden_additional_info(download_root, "nested/batch.hdf5", info)
|
|
70
68
|
|
|
71
69
|
success, frames = convert_batch(download_root, restored_root)
|
|
72
70
|
|
|
73
71
|
assert success == 1
|
|
74
72
|
assert frames == 1
|
|
75
|
-
output_path = restored_root / "nested" / "batch" / "
|
|
76
|
-
assert json.loads(output_path.read_text(encoding="utf-8")) ==
|
|
73
|
+
output_path = restored_root / "nested" / "batch" / "additional_info.json"
|
|
74
|
+
assert json.loads(output_path.read_text(encoding="utf-8")) == info
|
|
77
75
|
|
|
78
76
|
|
|
79
77
|
def test_standalone_hdf5_without_downloader_metadata_stays_compatible(tmp_path):
|
|
@@ -84,4 +82,4 @@ def test_standalone_hdf5_without_downloader_metadata_stays_compatible(tmp_path):
|
|
|
84
82
|
frames = convert_hdf5(source, restored)
|
|
85
83
|
|
|
86
84
|
assert frames == 1
|
|
87
|
-
assert not (restored / "
|
|
85
|
+
assert not (restored / "additional_info.json").exists()
|
|
@@ -265,7 +265,7 @@ def test_minimal_two_field_manifest_downloads(tmp_path, http_server):
|
|
|
265
265
|
assert (out / "ep.hdf5").read_bytes() == body
|
|
266
266
|
|
|
267
267
|
|
|
268
|
-
def
|
|
268
|
+
def test_duplicate_filename_download_and_additional_info_keep_first(tmp_path, http_server):
|
|
269
269
|
first_body = b"first-body"
|
|
270
270
|
second_body = b"second-body"
|
|
271
271
|
_seed(tmp_path, {"first.hdf5": first_body, "second.hdf5": second_body})
|
|
@@ -273,12 +273,12 @@ def test_duplicate_filename_download_and_instruction_keep_first(tmp_path, http_s
|
|
|
273
273
|
{
|
|
274
274
|
"download_url": http_server.url("first.hdf5"),
|
|
275
275
|
"file_name": "same.hdf5",
|
|
276
|
-
"
|
|
276
|
+
"additional_info": {"source": "第一条", "value": 1.1111111111111112e+54},
|
|
277
277
|
},
|
|
278
278
|
{
|
|
279
279
|
"download_url": http_server.url("second.hdf5"),
|
|
280
280
|
"file_name": "same.hdf5",
|
|
281
|
-
"
|
|
281
|
+
"additional_info": {"source": "第二条"},
|
|
282
282
|
},
|
|
283
283
|
]
|
|
284
284
|
manifest = Manifest.from_payload(payload)
|
|
@@ -290,11 +290,11 @@ def test_duplicate_filename_download_and_instruction_keep_first(tmp_path, http_s
|
|
|
290
290
|
assert report.total == 1
|
|
291
291
|
assert report.downloaded == 1
|
|
292
292
|
assert (out / "same.hdf5").read_bytes() == first_body
|
|
293
|
-
|
|
294
|
-
assert json.loads(
|
|
293
|
+
info_path = st.additional_info_path(out, "same.hdf5")
|
|
294
|
+
assert json.loads(info_path.read_text(encoding="utf-8")) == payload[0]["additional_info"]
|
|
295
295
|
|
|
296
296
|
|
|
297
|
-
def
|
|
297
|
+
def test_skipped_file_still_syncs_additional_info(tmp_path, http_server):
|
|
298
298
|
body = b"already-downloaded"
|
|
299
299
|
out = tmp_path / "out"
|
|
300
300
|
out.mkdir()
|
|
@@ -303,26 +303,26 @@ def test_skipped_file_still_syncs_task_instruction(tmp_path, http_server):
|
|
|
303
303
|
"download_url": http_server.url("unused.hdf5"),
|
|
304
304
|
"file_name": "ep.hdf5",
|
|
305
305
|
"file_size": len(body),
|
|
306
|
-
"
|
|
306
|
+
"additional_info": {"mode": "skip"},
|
|
307
307
|
}])
|
|
308
308
|
|
|
309
309
|
report = DownloadManager(out, backend="http").run(manifest)
|
|
310
310
|
|
|
311
311
|
assert report.ok
|
|
312
312
|
assert report.skipped == 1
|
|
313
|
-
|
|
314
|
-
assert json.loads(
|
|
313
|
+
info_path = st.additional_info_path(out, "ep.hdf5")
|
|
314
|
+
assert json.loads(info_path.read_text(encoding="utf-8")) == {"mode": "skip"}
|
|
315
315
|
|
|
316
316
|
|
|
317
|
-
def
|
|
317
|
+
def test_failed_or_corrupt_file_does_not_keep_additional_info(tmp_path, http_server):
|
|
318
318
|
out = tmp_path / "out"
|
|
319
|
-
stale_path = st.
|
|
319
|
+
stale_path = st.additional_info_path(out, "ep.hdf5")
|
|
320
320
|
stale_path.parent.mkdir(parents=True)
|
|
321
321
|
stale_path.write_text('{"stale": true}', encoding="utf-8")
|
|
322
322
|
failed = Manifest.from_payload([{
|
|
323
323
|
"download_url": http_server.url("missing.hdf5"),
|
|
324
324
|
"file_name": "ep.hdf5",
|
|
325
|
-
"
|
|
325
|
+
"additional_info": {"new": True},
|
|
326
326
|
}])
|
|
327
327
|
|
|
328
328
|
failed_report = DownloadManager(out, backend="http").run(failed)
|
|
@@ -337,7 +337,7 @@ def test_failed_or_corrupt_file_does_not_keep_task_instruction(tmp_path, http_se
|
|
|
337
337
|
"file_name": "ep.hdf5",
|
|
338
338
|
"file_size": len(body),
|
|
339
339
|
"sha256": "0" * 64,
|
|
340
|
-
"
|
|
340
|
+
"additional_info": {"new": True},
|
|
341
341
|
}])
|
|
342
342
|
|
|
343
343
|
corrupt_report = DownloadManager(out, backend="http", deep=True).run(corrupt)
|
|
@@ -346,11 +346,11 @@ def test_failed_or_corrupt_file_does_not_keep_task_instruction(tmp_path, http_se
|
|
|
346
346
|
assert not stale_path.exists()
|
|
347
347
|
|
|
348
348
|
|
|
349
|
-
def
|
|
349
|
+
def test_missing_additional_info_clears_stale_hidden_metadata(tmp_path, http_server):
|
|
350
350
|
body = b"fresh-body"
|
|
351
351
|
_seed(tmp_path, {"ep.hdf5": body})
|
|
352
352
|
out = tmp_path / "out"
|
|
353
|
-
stale_path = st.
|
|
353
|
+
stale_path = st.additional_info_path(out, "ep.hdf5")
|
|
354
354
|
stale_path.parent.mkdir(parents=True)
|
|
355
355
|
stale_path.write_text('{"stale": true}', encoding="utf-8")
|
|
356
356
|
manifest = Manifest.from_payload([{
|
|
@@ -48,48 +48,88 @@ def test_manifest_from_files_wrapper_native():
|
|
|
48
48
|
assert m.files[0].presignedUrl.startswith("https")
|
|
49
49
|
|
|
50
50
|
|
|
51
|
-
def
|
|
52
|
-
|
|
51
|
+
def test_additional_info_accepts_native_and_canonical_names():
|
|
52
|
+
info = {
|
|
53
53
|
"aaa": "111",
|
|
54
54
|
"aaaa": 222,
|
|
55
55
|
"ewrwrwe": 1.1111111111111112e+54,
|
|
56
56
|
}
|
|
57
57
|
native = Manifest.from_payload({
|
|
58
|
-
"files": [{**_native_file(), "
|
|
58
|
+
"files": [{**_native_file(), "additional_info": info}],
|
|
59
59
|
})
|
|
60
60
|
canonical = Manifest.from_payload({
|
|
61
|
-
"files": [{**_native_file(), "
|
|
61
|
+
"files": [{**_native_file(), "additionalInfo": info}],
|
|
62
62
|
})
|
|
63
63
|
|
|
64
|
-
assert native.files[0].
|
|
65
|
-
assert canonical.files[0].
|
|
66
|
-
|
|
64
|
+
assert native.files[0].additionalInfo == info
|
|
65
|
+
assert canonical.files[0].additionalInfo == info
|
|
66
|
+
for manifest in (native, canonical):
|
|
67
|
+
dumped_file = json.loads(manifest.model_dump_json())["files"][0]
|
|
68
|
+
assert dumped_file["additionalInfo"] == info
|
|
69
|
+
assert "additional_info" not in dumped_file
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@pytest.mark.parametrize(
|
|
73
|
+
("native_info", "canonical_info"),
|
|
74
|
+
[
|
|
75
|
+
({"same": 1}, {"same": 1}),
|
|
76
|
+
({"snake": 1}, {"camel": 2}),
|
|
77
|
+
],
|
|
78
|
+
ids=["equal-values", "conflicting-values"],
|
|
79
|
+
)
|
|
80
|
+
def test_additional_info_aliases_cannot_be_used_together(
|
|
81
|
+
native_info,
|
|
82
|
+
canonical_info,
|
|
83
|
+
):
|
|
84
|
+
with pytest.raises(ValidationError, match="cannot be used together"):
|
|
85
|
+
Manifest.from_payload({
|
|
86
|
+
"files": [{
|
|
87
|
+
**_native_file(),
|
|
88
|
+
"additional_info": native_info,
|
|
89
|
+
"additionalInfo": canonical_info,
|
|
90
|
+
}],
|
|
91
|
+
})
|
|
67
92
|
|
|
68
93
|
|
|
69
|
-
def
|
|
94
|
+
def test_additional_info_must_be_an_object():
|
|
70
95
|
with pytest.raises(ValidationError):
|
|
71
96
|
Manifest.from_payload({
|
|
72
|
-
"files": [{**_native_file(), "
|
|
97
|
+
"files": [{**_native_file(), "additional_info": ["not", "an", "object"]}],
|
|
73
98
|
})
|
|
74
99
|
|
|
75
100
|
|
|
76
|
-
|
|
101
|
+
@pytest.mark.parametrize("legacy_field", ["task_instruction", "taskInstruction"])
|
|
102
|
+
def test_removed_task_instruction_fields_are_rejected(legacy_field):
|
|
103
|
+
with pytest.raises(ValidationError, match="use 'additional_info' instead"):
|
|
104
|
+
Manifest.from_payload({
|
|
105
|
+
"files": [{**_native_file(), legacy_field: {"removed": True}}],
|
|
106
|
+
})
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def test_unrelated_extension_fields_remain_allowed():
|
|
110
|
+
manifest = Manifest.from_payload({
|
|
111
|
+
"files": [{**_native_file(), "additional_info": {"source": "native"}, "future": 1}],
|
|
112
|
+
})
|
|
113
|
+
assert manifest.files[0].model_extra == {"expires_at": "2026-07-07T04:04:14+00:00", "expires_in": 3600, "future": 1}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def test_duplicate_local_path_keeps_first_file_and_additional_info():
|
|
77
117
|
first = {
|
|
78
118
|
**_native_file("first/source.hdf5", url="https://host/first/source.hdf5"),
|
|
79
119
|
"file_name": "same.hdf5",
|
|
80
|
-
"
|
|
120
|
+
"additional_info": {"source": "first"},
|
|
81
121
|
}
|
|
82
122
|
second = {
|
|
83
123
|
**_native_file("second/source.hdf5", url="https://host/second/source.hdf5"),
|
|
84
124
|
"file_name": "same.hdf5",
|
|
85
|
-
"
|
|
125
|
+
"additional_info": {"source": "second"},
|
|
86
126
|
}
|
|
87
127
|
|
|
88
128
|
manifest = Manifest.from_payload({"files": [first, second]})
|
|
89
129
|
|
|
90
130
|
assert len(manifest.files) == 1
|
|
91
131
|
assert manifest.files[0].objectKey == "first/source.hdf5"
|
|
92
|
-
assert manifest.files[0].
|
|
132
|
+
assert manifest.files[0].additionalInfo == {"source": "first"}
|
|
93
133
|
|
|
94
134
|
|
|
95
135
|
def test_from_payload_accepts_json_string_and_list_string():
|
dexverse_cli-0.0.2/648584.hdf5
DELETED
|
Binary file
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|