dexverse-cli 0.0.4.post1__tar.gz → 0.0.4.post2__tar.gz

{dexverse_cli-0.0.4.post1 → dexverse_cli-0.0.4.post2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: dexverse-cli
-Version: 0.0.4.post1
+Version: 0.0.4.post2
 Summary: DexVerse dataset downloader for S3-compatible robot training datasets
 Author: DexVerse
 License: Apache-2.0
@@ -161,7 +161,8 @@ dexverse-dl convert --input-dir /data/dexverse -o /data/dexverse/restored
 
 `task_instruction.json` 的内容就是 manifest 中的对象本身，不增加外层包装。
 下载器会把转换所需的映射保存在 `.dexverse/task-instructions/`，无需为
-`convert` 增加额外参数。独立来源且没有该隐藏元数据的 HDF5 仍按原行为还原。
+`convert` 增加额外参数。设备侧 `data_protocol` 不会写入算法侧
+`task_instruction.json`；独立来源且没有该隐藏元数据的 HDF5 仍按原行为还原。
 
 该命令依赖 `h5py` / `numpy` / `av` / `pillow`，已列入核心依赖，安装时随包装上。
 

{dexverse_cli-0.0.4.post1 → dexverse_cli-0.0.4.post2}/README.md

@@ -131,7 +131,8 @@ dexverse-dl convert --input-dir /data/dexverse -o /data/dexverse/restored
 
 `task_instruction.json` 的内容就是 manifest 中的对象本身，不增加外层包装。
 下载器会把转换所需的映射保存在 `.dexverse/task-instructions/`，无需为
-`convert` 增加额外参数。独立来源且没有该隐藏元数据的 HDF5 仍按原行为还原。
+`convert` 增加额外参数。设备侧 `data_protocol` 不会写入算法侧
+`task_instruction.json`；独立来源且没有该隐藏元数据的 HDF5 仍按原行为还原。
 
 该命令依赖 `h5py` / `numpy` / `av` / `pillow`，已列入核心依赖，安装时随包装上。
 

{dexverse_cli-0.0.4.post1 → dexverse_cli-0.0.4.post2}/dexverse/convert/hdf5_to_raw.py

@@ -31,6 +31,7 @@ import h5py
 import numpy as np
 
 from ..downloader import state as download_state
+from ..manifest import algorithm_task_instruction
 # 复用同包子模块的解码能力
 from .image_video_encode import decode_mp4_bytes_to_rgb_images
 
@@ -68,7 +69,12 @@ def _sync_restored_task_instruction(
         if not isinstance(task_instruction, dict):
             raise ValueError(f"task instruction must be a JSON object: {source}")
         destination.write_text(
-            json.dumps(task_instruction, ensure_ascii=False, indent=2) + "\n",
+            json.dumps(
+                algorithm_task_instruction(task_instruction),
+                ensure_ascii=False,
+                indent=2,
+            )
+            + "\n",
             encoding="utf-8",
         )
     else:

{dexverse_cli-0.0.4.post1 → dexverse_cli-0.0.4.post2}/dexverse/downloader/manager.py

@@ -20,7 +20,7 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Dict, List, Optional, Union
 
-from ..manifest import FileSpec, Manifest, resolve_dest
+from ..manifest import FileSpec, Manifest, algorithm_task_instruction, resolve_dest
 from . import state as st
 from .base import Backend, ProgressReporter
 from .http_backend import HttpBackend, _HTTP_BACKEND_NAMES
@@ -62,7 +62,7 @@ def _sync_task_instructions(
             instruction_path.parent.mkdir(parents=True, exist_ok=True)
             instruction_path.write_text(
                 json.dumps(
-                    file_spec.additionalInfo,
+                    algorithm_task_instruction(file_spec.additionalInfo),
                     ensure_ascii=False,
                     indent=2,
                 )

{dexverse_cli-0.0.4.post1 → dexverse_cli-0.0.4.post2}/dexverse/manifest.py

@@ -22,6 +22,13 @@ from typing import Any, Dict, List, Optional
 from pydantic import AliasChoices, BaseModel, ConfigDict, Field, model_validator
 
 
+def algorithm_task_instruction(additional_info: Dict[str, Any]) -> Dict[str, Any]:
+    """Return algorithm-facing task metadata without device-only protocol data."""
+    task_instruction = dict(additional_info)
+    task_instruction.pop("data_protocol", None)
+    return task_instruction
+
+
 # --------------------------------------------------------------------------- #
 # Path helpers
 # --------------------------------------------------------------------------- #

{dexverse_cli-0.0.4.post1 → dexverse_cli-0.0.4.post2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "dexverse-cli"
-version = "0.0.4.post1"
+version = "0.0.4.post2"
 description = "DexVerse dataset downloader for S3-compatible robot training datasets"
 readme = "README.md"
 requires-python = ">=3.9"

{dexverse_cli-0.0.4.post1 → dexverse_cli-0.0.4.post2}/tests/test_convert_additional_info.py

@@ -38,7 +38,7 @@ def test_convert_single_file_writes_and_updates_raw_task_instruction(tmp_path):
     info_path = _write_hidden_task_instruction(
         download_root,
         "session.hdf5",
-        {"aaa": "中文", "number": 1.1111111111111112e+54},
+        {"aaa": "中文", "number": 1.1111111111111112e+54, "data_protocol": "W1"},
     )
 
     frames = convert_hdf5(source, restored)
@@ -92,10 +92,22 @@ def test_convert_bridges_additional_info_state_to_legacy_output(tmp_path):
     _write_minimal_hdf5(source)
     bridge_path = st.additional_info_path(download_root, "session.hdf5")
     bridge_path.parent.mkdir(parents=True)
-    bridge_path.write_text('{"from": "32ffa8e"}\n', encoding="utf-8")
+    bridge_path.write_text('{"from": "32ffa8e", "data_protocol": "SIM"}\n', encoding="utf-8")
 
     convert_hdf5(source, restored)
 
     assert json.loads((restored / "task_instruction.json").read_text(encoding="utf-8")) == {
         "from": "32ffa8e"
     }
+
+
+def test_convert_data_protocol_only_writes_empty_task_instruction(tmp_path):
+    download_root = tmp_path / "download"
+    source = download_root / "session.hdf5"
+    restored = tmp_path / "restored" / "session"
+    _write_minimal_hdf5(source)
+    _write_hidden_task_instruction(download_root, "session.hdf5", {"data_protocol": "UMI"})
+
+    convert_hdf5(source, restored)
+
+    assert json.loads((restored / "task_instruction.json").read_text(encoding="utf-8")) == {}

{dexverse_cli-0.0.4.post1 → dexverse_cli-0.0.4.post2}/tests/test_http_backend.py

@@ -273,7 +273,11 @@ def test_duplicate_filename_download_and_additional_info_keep_first(tmp_path, ht
         {
             "download_url": http_server.url("first.hdf5"),
             "file_name": "same.hdf5",
-            "additional_info": {"source": "第一条", "value": 1.1111111111111112e+54},
+            "additional_info": {
+                "source": "第一条",
+                "value": 1.1111111111111112e+54,
+                "data_protocol": "W1",
+            },
         },
         {
             "download_url": http_server.url("second.hdf5"),
@@ -291,7 +295,11 @@ def test_duplicate_filename_download_and_additional_info_keep_first(tmp_path, ht
     assert report.downloaded == 1
     assert (out / "same.hdf5").read_bytes() == first_body
     instruction_path = st.task_instruction_path(out, "same.hdf5")
-    assert json.loads(instruction_path.read_text(encoding="utf-8")) == payload[0]["additional_info"]
+    assert json.loads(instruction_path.read_text(encoding="utf-8")) == {
+        "source": "第一条",
+        "value": 1.1111111111111112e+54,
+    }
+    assert manifest.files[0].additionalInfo["data_protocol"] == "W1"
     assert not st.additional_info_path(out, "same.hdf5").exists()
 
 
@@ -304,7 +312,7 @@ def test_legacy_manifest_metadata_still_syncs_to_legacy_path(tmp_path, http_serv
         "download_url": http_server.url("unused.hdf5"),
         "file_name": "ep.hdf5",
         "file_size": len(body),
-        "task_instruction": {"mode": "legacy"},
+        "task_instruction": {"mode": "legacy", "data_protocol": "SIM"},
     }])
 
     report = DownloadManager(out, backend="http").run(manifest)
@@ -315,6 +323,24 @@ def test_legacy_manifest_metadata_still_syncs_to_legacy_path(tmp_path, http_serv
     assert not st.additional_info_path(out, "ep.hdf5").exists()
 
 
+def test_data_protocol_only_writes_an_empty_task_instruction(tmp_path, http_server):
+    body = b"already-downloaded"
+    out = tmp_path / "out"
+    out.mkdir()
+    (out / "ep.hdf5").write_bytes(body)
+    manifest = Manifest.from_payload([{
+        "download_url": http_server.url("unused.hdf5"),
+        "file_name": "ep.hdf5",
+        "file_size": len(body),
+        "additional_info": {"data_protocol": "UMI"},
+    }])
+
+    report = DownloadManager(out, backend="http").run(manifest)
+
+    assert report.ok
+    assert json.loads(st.task_instruction_path(out, "ep.hdf5").read_text(encoding="utf-8")) == {}
+
+
 def test_skipped_file_still_syncs_additional_info(tmp_path, http_server):
     body = b"already-downloaded"
     out = tmp_path / "out"