dexverse-cli 0.0.2__tar.gz

dexverse_cli-0.0.2/.dockerignore

@@ -0,0 +1,14 @@
+.venv/
+venv/
+__pycache__/
+*.pyc
+.pytest_cache/
+.git/
+.gitignore
+tests/
+examples/
+*.md
+DexVerse_Downloader_S3_Final_*.md
+download-state.json
+*.s5cmd
+.dexverse/

dexverse_cli-0.0.2/.gitignore

@@ -0,0 +1,41 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+*.whl
+
+# Virtual envs
+.venv/
+venv/
+env/
+
+# Test / coverage
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Secrets — NEVER commit (see doc section 13.1)
+.env
+.env.*
+config.toml
+dexverse/config.toml
+
+# Real manifests with presigned URLs / S3 creds (regenerated per download).
+# examples/manifest*.json are tracked templates with placeholder URLs.
+/manifest.json
+/manifest*.json
+
+# Local download state / artifacts
+.dexverse/
+*.s5cmd
+download-state.json
+download.log
+
+# Editor / OS
+.idea/
+.vscode/
+.DS_Store
+Thumbs.db

dexverse_cli-0.0.2/Dockerfile

@@ -0,0 +1,37 @@
+# syntax=docker/dockerfile:1
+# DexVerse downloader — bundles Python + dexverse + s5cmd (doc §19).
+# Supports all three backends: s5cmd (default), python (boto3), http (presigned URLs).
+#
+#   docker build -t dexverse/dexverse-dl:latest .
+#   docker run --rm -v /data:/out dexverse/dexverse-dl:latest \
+#       download --manifest /manifest.json --out /out --backend http
+#
+# Or with a task id (manifest fetched from the backend inside the container):
+#   docker run --rm -v /data:/out -e DEXVERSE_TOKEN=xxx \
+#       dexverse/dexverse-dl:latest download --task dl_xxx --out /out \
+#       --server https://dexverse.example.com --backend http
+
+FROM python:3.11-slim AS base
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1
+
+# s5cmd (for the high-throughput s5cmd backend; harmless if you only use http/python)
+ARG S5CMD_VERSION=v2.3.0
+ADD https://github.com/peak/s5cmd/releases/download/${S5CMD_VERSION}/s5cmd_${S5CMD_VERSION}_Linux-64bit.tar.gz /tmp/s5cmd.tar.gz
+RUN tar -xzf /tmp/s5cmd.tar.gz -C /usr/local/bin s5cmd && rm /tmp/s5cmd.tar.gz
+
+# The package version comes from pyproject.toml. Keep exactly one built wheel
+# in dist/ so Docker never needs a duplicated, hard-coded version number.
+COPY dist/dexverse-*.whl /tmp/wheels/
+RUN set -- /tmp/wheels/dexverse-*.whl; \
+    if [ "$#" -ne 1 ]; then \
+        echo "expected exactly one dexverse wheel in dist/, found $#"; \
+        exit 1; \
+    fi; \
+    pip install --no-cache-dir "${1}[s3]"
+
+WORKDIR /work
+ENTRYPOINT ["dexverse-dl"]
+CMD ["--help"]

dexverse_cli-0.0.2/PKG-INFO

@@ -0,0 +1,198 @@
+Metadata-Version: 2.4
+Name: dexverse-cli
+Version: 0.0.2
+Summary: DexVerse dataset downloader for S3-compatible robot training datasets
+Author: DexVerse
+License: Apache-2.0
+Keywords: dataset,dexverse,downloader,minio,robotics,s3,s5cmd
+Requires-Python: >=3.9
+Requires-Dist: av>=12
+Requires-Dist: h5py>=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: numpy>=1.26
+Requires-Dist: pillow>=10
+Requires-Dist: pydantic>=2.0
+Requires-Dist: rich>=13.0
+Requires-Dist: tomlkit>=0.12
+Requires-Dist: typer>=0.12
+Provides-Extra: dev
+Requires-Dist: boto3>=1.34; extra == 'dev'
+Requires-Dist: fastapi>=0.110; extra == 'dev'
+Requires-Dist: httpx>=0.27; extra == 'dev'
+Requires-Dist: moto[s3]>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Provides-Extra: mock
+Requires-Dist: fastapi>=0.110; extra == 'mock'
+Requires-Dist: uvicorn>=0.27; extra == 'mock'
+Provides-Extra: s3
+Requires-Dist: boto3>=1.34; extra == 's3'
+Description-Content-Type: text/markdown
+
+# DexVerse Downloader (`dexverse`)
+
+Presigned-URL batch downloader for robot-training datasets.
+
+```text
+manifest -> dexverse-dl -> httpx -> presigned URLs -> local file landing
+```
+
+The primary input is the DexVerse backend's raw presigned-URL JSON. For those
+payloads, `storage_path` is the remote object key and `file_name` is the local
+landing path under `--out`.
+
+Canonical manifests are still supported. Their local paths come from
+`relativePath`, falling back to `originalPath`, then `objectKey`:
+
+```text
+localPath = out / (file.relativePath || file.originalPath || file.objectKey)
+```
+
+## Install
+
+```bash
+pip install "dexverse[s3,mock]"          # boto3 + fastapi mock server
+# optional legacy S3 backend:
+#   install s5cmd (https://github.com/peak/s5cmd) and put it on PATH
+```
+
+Development:
+
+```bash
+uv venv && . .venv/bin/activate          # Windows: .venv\Scripts\activate
+uv pip install -e ".[dev]"
+```
+
+## Quick Start
+
+If your backend issues short-lived presigned URLs per file, save that response
+as `download.json` and run:
+
+```bash
+dexverse-dl download --manifest ./download.json --out D:\DexVerse
+```
+
+No login, no `AWS_*` credentials, no s5cmd, no backend flag, and no TLS flag are
+required for the common internal/self-signed endpoint case. The default backend
+is `http`, and TLS verification is disabled by default for http/presigned
+downloads. Use `--secure` when the endpoint has a valid certificate and you want
+certificate validation.
+
+Raw backend output is accepted directly:
+
+```json
+[{
+  "download_url": "https://host/bucket/datasets/.../20260429_180359.hdf5?X-Amz-Signature=...",
+  "storage_bucket": "dexverse",
+  "storage_path": "datasets/.../20260429_180359.hdf5",
+  "file_name": "20260429_180359.hdf5",
+  "file_size": 14994636,
+  "task_instruction": {
+    "aaa": "111",
+    "aaaa": 222,
+    "ewrwrwe": 1.1111111111111112e+54
+  }
+}]
+```
+
+Result:
+
+```text
+D:\DexVerse\20260429_180359.hdf5
+```
+
+`task_instruction` is optional and must be a JSON object. If multiple manifest
+items resolve to the same local path (for example, repeated `file_name`
+values), the first item wins and later duplicates are ignored.
+
+Include `file_size` (and preferably `sha256`) for large files. The HTTP backend
+also learns the total size from `Content-Length`/`Content-Range`, but manifest
+metadata lets the CLI verify an existing local file before making a request.
+
+Transient timeouts and HTTP 408/425/429/5xx responses are retried five times by
+default. Bytes already received stay in a sibling `<file>.part`; retries and
+later CLI runs request the remaining bytes with HTTP Range, while the final
+destination path only appears after the size check succeeds. The object server
+must support Range requests for efficient resume.
+
+Presigned URLs expire, often after 1 hour. Have the backend regenerate the
+manifest right before each download; expired URLs surface as per-file `HTTP 403`
+errors. After generating a fresh `download.json`, run the same command with the
+same `--out` path to continue from the retained `.part`.
+
+## CLI
+
+```bash
+dexverse-dl login   --server https://dexverse.xxx.com [--token TOKEN]
+dexverse-dl download --manifest ./download.json --out /data/dexverse [--parallel 16] [--secure] [--backend http|python|s5cmd]
+dexverse-dl download --manifest ./download.json --out /data/dexverse [--http-read-timeout 120] [--http-retries 5]
+dexverse-dl download --task dl_xxx --out /data/dexverse [--server URL]
+dexverse-dl verify  --manifest PATH --out /data/dexverse [--deep]
+dexverse-dl repair  --manifest PATH --out /data/dexverse [--backend http|python|s5cmd]
+```
+
+### 还原 HDF5 → 可训练数据
+
+下载完成后，把落盘的 `.hdf5` 还原为原始录制结构（逐帧 JPEG + `pose_record_*.json`
++ `metadata.jsonl`）。如果对应 manifest 带有 `task_instruction`，还会自动生成
+`task_instruction.json`：
+
+```bash
+# 单文件
+dexverse-dl convert -i /data/dexverse/20260429_180359.hdf5 -o /data/dexverse/restored
+# 批量（递归处理目录下所有 *.hdf5，保持目录结构）
+dexverse-dl convert --input-dir /data/dexverse -o /data/dexverse/restored
+```
+
+还原输出形如：
+
+```text
+/data/dexverse/restored/20260429_180359/
+  head/left/000000.jpg ...
+  head/right/000000.jpg ...
+  hand/left/000000.jpg ...
+  hand/right/000000.jpg ...
+  pose_record_20260429_180359.json
+  metadata.jsonl
+  task_instruction.json
+```
+
+`task_instruction.json` 的内容就是 manifest 中的对象本身，不增加外层包装。
+下载器会把转换所需的映射保存在 `.dexverse/task-instructions/`，无需为
+`convert` 增加额外参数。独立来源且没有该隐藏元数据的 HDF5 仍按原行为还原。
+
+该命令依赖 `h5py` / `numpy` / `av` / `pillow`，已列入核心依赖，安装时随包装上。
+
+## Python SDK
+
+```python
+from dexverse import DexVerseClient, download_manifest
+
+client = DexVerseClient(server="https://dexverse.xxx.com", token="xxx")
+local_root = client.download_task(task_id="dl_xxx", out="/data/dexverse", parallel=16)
+
+# or from a local manifest file
+download_manifest(manifest_path="./download.json", out="/data/dexverse")
+```
+
+## Layout
+
+State files live under `<out>/.dexverse/` and never mix into the dataset tree:
+
+```text
+/data/dexverse/
+  .dexverse/task-instructions/
+    20260429_180359.hdf5.json
+  .dexverse/tasks/<taskId>/
+    manifest.json
+    download-state.json
+    download.log
+  20260429_180359.hdf5
+```
+
+## Backends
+
+| backend  | engine            | notes                                   |
+|----------|-------------------|-----------------------------------------|
+| `http`   | httpx             | default; presigned URLs, no creds/login |
+| `python` | boto3             | legacy S3 fallback                      |
+| `s5cmd`  | peak/s5cmd binary | legacy S3 high-throughput backend       |

dexverse_cli-0.0.2/README.md

@@ -0,0 +1,168 @@
+# DexVerse Downloader (`dexverse`)
+
+Presigned-URL batch downloader for robot-training datasets.
+
+```text
+manifest -> dexverse-dl -> httpx -> presigned URLs -> local file landing
+```
+
+The primary input is the DexVerse backend's raw presigned-URL JSON. For those
+payloads, `storage_path` is the remote object key and `file_name` is the local
+landing path under `--out`.
+
+Canonical manifests are still supported. Their local paths come from
+`relativePath`, falling back to `originalPath`, then `objectKey`:
+
+```text
+localPath = out / (file.relativePath || file.originalPath || file.objectKey)
+```
+
+## Install
+
+```bash
+pip install "dexverse[s3,mock]"          # boto3 + fastapi mock server
+# optional legacy S3 backend:
+#   install s5cmd (https://github.com/peak/s5cmd) and put it on PATH
+```
+
+Development:
+
+```bash
+uv venv && . .venv/bin/activate          # Windows: .venv\Scripts\activate
+uv pip install -e ".[dev]"
+```
+
+## Quick Start
+
+If your backend issues short-lived presigned URLs per file, save that response
+as `download.json` and run:
+
+```bash
+dexverse-dl download --manifest ./download.json --out D:\DexVerse
+```
+
+No login, no `AWS_*` credentials, no s5cmd, no backend flag, and no TLS flag are
+required for the common internal/self-signed endpoint case. The default backend
+is `http`, and TLS verification is disabled by default for http/presigned
+downloads. Use `--secure` when the endpoint has a valid certificate and you want
+certificate validation.
+
+Raw backend output is accepted directly:
+
+```json
+[{
+  "download_url": "https://host/bucket/datasets/.../20260429_180359.hdf5?X-Amz-Signature=...",
+  "storage_bucket": "dexverse",
+  "storage_path": "datasets/.../20260429_180359.hdf5",
+  "file_name": "20260429_180359.hdf5",
+  "file_size": 14994636,
+  "task_instruction": {
+    "aaa": "111",
+    "aaaa": 222,
+    "ewrwrwe": 1.1111111111111112e+54
+  }
+}]
+```
+
+Result:
+
+```text
+D:\DexVerse\20260429_180359.hdf5
+```
+
+`task_instruction` is optional and must be a JSON object. If multiple manifest
+items resolve to the same local path (for example, repeated `file_name`
+values), the first item wins and later duplicates are ignored.
+
+Include `file_size` (and preferably `sha256`) for large files. The HTTP backend
+also learns the total size from `Content-Length`/`Content-Range`, but manifest
+metadata lets the CLI verify an existing local file before making a request.
+
+Transient timeouts and HTTP 408/425/429/5xx responses are retried five times by
+default. Bytes already received stay in a sibling `<file>.part`; retries and
+later CLI runs request the remaining bytes with HTTP Range, while the final
+destination path only appears after the size check succeeds. The object server
+must support Range requests for efficient resume.
+
+Presigned URLs expire, often after 1 hour. Have the backend regenerate the
+manifest right before each download; expired URLs surface as per-file `HTTP 403`
+errors. After generating a fresh `download.json`, run the same command with the
+same `--out` path to continue from the retained `.part`.
+
+## CLI
+
+```bash
+dexverse-dl login   --server https://dexverse.xxx.com [--token TOKEN]
+dexverse-dl download --manifest ./download.json --out /data/dexverse [--parallel 16] [--secure] [--backend http|python|s5cmd]
+dexverse-dl download --manifest ./download.json --out /data/dexverse [--http-read-timeout 120] [--http-retries 5]
+dexverse-dl download --task dl_xxx --out /data/dexverse [--server URL]
+dexverse-dl verify  --manifest PATH --out /data/dexverse [--deep]
+dexverse-dl repair  --manifest PATH --out /data/dexverse [--backend http|python|s5cmd]
+```
+
+### 还原 HDF5 → 可训练数据
+
+下载完成后，把落盘的 `.hdf5` 还原为原始录制结构（逐帧 JPEG + `pose_record_*.json`
++ `metadata.jsonl`）。如果对应 manifest 带有 `task_instruction`，还会自动生成
+`task_instruction.json`：
+
+```bash
+# 单文件
+dexverse-dl convert -i /data/dexverse/20260429_180359.hdf5 -o /data/dexverse/restored
+# 批量（递归处理目录下所有 *.hdf5，保持目录结构）
+dexverse-dl convert --input-dir /data/dexverse -o /data/dexverse/restored
+```
+
+还原输出形如：
+
+```text
+/data/dexverse/restored/20260429_180359/
+  head/left/000000.jpg ...
+  head/right/000000.jpg ...
+  hand/left/000000.jpg ...
+  hand/right/000000.jpg ...
+  pose_record_20260429_180359.json
+  metadata.jsonl
+  task_instruction.json
+```
+
+`task_instruction.json` 的内容就是 manifest 中的对象本身，不增加外层包装。
+下载器会把转换所需的映射保存在 `.dexverse/task-instructions/`，无需为
+`convert` 增加额外参数。独立来源且没有该隐藏元数据的 HDF5 仍按原行为还原。
+
+该命令依赖 `h5py` / `numpy` / `av` / `pillow`，已列入核心依赖，安装时随包装上。
+
+## Python SDK
+
+```python
+from dexverse import DexVerseClient, download_manifest
+
+client = DexVerseClient(server="https://dexverse.xxx.com", token="xxx")
+local_root = client.download_task(task_id="dl_xxx", out="/data/dexverse", parallel=16)
+
+# or from a local manifest file
+download_manifest(manifest_path="./download.json", out="/data/dexverse")
+```
+
+## Layout
+
+State files live under `<out>/.dexverse/` and never mix into the dataset tree:
+
+```text
+/data/dexverse/
+  .dexverse/task-instructions/
+    20260429_180359.hdf5.json
+  .dexverse/tasks/<taskId>/
+    manifest.json
+    download-state.json
+    download.log
+  20260429_180359.hdf5
+```
+
+## Backends
+
+| backend  | engine            | notes                                   |
+|----------|-------------------|-----------------------------------------|
+| `http`   | httpx             | default; presigned URLs, no creds/login |
+| `python` | boto3             | legacy S3 fallback                      |
+| `s5cmd`  | peak/s5cmd binary | legacy S3 high-throughput backend       |

dexverse_cli-0.0.2/dexverse/__init__.py

@@ -0,0 +1,54 @@
+"""DexVerse Downloader — S3-compatible dataset downloader.
+
+Public API:
+
+    from dexverse import DexVerseClient, download_manifest
+    from dexverse import Manifest, FileSpec
+"""
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
+
+import tomlkit
+
+from .client import DexVerseClient, DexVerseError, download_manifest
+from .manifest import (
+    FileSpec,
+    Manifest,
+    Storage,
+    StorageCredentials,
+    DownloadOptions,
+    resolve_dest,
+    resolve_relpath,
+)
+
+def _resolve_version() -> str:
+    """Read the single source version in development, package metadata otherwise."""
+    pyproject = Path(__file__).resolve().parent.parent / "pyproject.toml"
+    if pyproject.is_file():
+        document = tomlkit.parse(pyproject.read_text(encoding="utf-8"))
+        project_version = document.get("project", {}).get("version")
+        if isinstance(project_version, str):
+            return project_version
+    try:
+        return version("dexverse")
+    except PackageNotFoundError:
+        return "0.0.0+unknown"
+
+
+__version__ = _resolve_version()
+
+__all__ = [
+    "DexVerseClient",
+    "DexVerseError",
+    "download_manifest",
+    "Manifest",
+    "FileSpec",
+    "Storage",
+    "StorageCredentials",
+    "DownloadOptions",
+    "resolve_dest",
+    "resolve_relpath",
+    "__version__",
+]