dexverse-cli 0.0.2__py3-none-any.whl

dexverse/mock/server.py

@@ -0,0 +1,211 @@
+"""Mock backend (doc §9) — simulates the platform so the CLI/SDK can run the
+real flow without a production server.
+
+    GET  /api/download-tasks/{task_id}/manifest   (doc §9.3, §12.2)
+    POST /api/dev/download-tasks                  (doc §8.3 — create from files)
+
+Env vars (doc §9.2):
+
+    DEXVERSE_DEV_TOKEN            default "dev-token"
+    DEXVERSE_S3_ENDPOINT
+    DEXVERSE_S3_REGION            default "us-east-1"
+    DEXVERSE_S3_ACCESS_KEY
+    DEXVERSE_S3_SECRET_KEY
+    DEXVERSE_S3_SESSION_TOKEN     optional
+    DEXVERSE_MOCK_TASKS           path to tasks json (default ./mock_tasks.json)
+
+Run:
+
+    dexverse-mock-server --port 18080
+    # (binds 127.0.0.1 by default — dev endpoints must not be exposed publicly,
+    #  doc §13.1 item 4. Use --host 0.0.0.0 only inside a trusted intranet.)
+"""
+from __future__ import annotations
+
+import json
+import os
+import threading
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from fastapi import FastAPI, Header, HTTPException
+from pydantic import BaseModel
+
+TASKS_PATH = Path(os.getenv("DEXVERSE_MOCK_TASKS", str(Path(__file__).parent / "mock_tasks.json")))
+
+_lock = threading.Lock()
+_tasks: Dict[str, Dict[str, Any]] = {}
+
+
+def _load_tasks() -> Dict[str, Dict[str, Any]]:
+    global _tasks
+    if not _tasks:
+        if TASKS_PATH.exists():
+            _tasks = json.loads(TASKS_PATH.read_text(encoding="utf-8"))
+        else:
+            _tasks = {}
+    return _tasks
+
+
+def _check_auth(authorization: Optional[str]) -> None:
+    expected = os.getenv("DEXVERSE_DEV_TOKEN", "dev-token")
+    if not authorization:
+        raise HTTPException(status_code=401, detail="Missing Authorization header")
+    if authorization != f"Bearer {expected}":
+        raise HTTPException(status_code=403, detail="Invalid token")
+
+
+def _truthy(name: str) -> bool:
+    return os.getenv(name, "").lower() in ("1", "true", "yes", "on")
+
+
+def _presign_client():
+    """A boto3 client for real presigning, or None if not configured."""
+    if not _truthy("DEXVERSE_MOCK_PRESIGN"):
+        return None
+    import boto3
+    from botocore.config import Config
+    endpoint = os.getenv("DEXVERSE_S3_ENDPOINT")
+    ak = os.getenv("DEXVERSE_S3_ACCESS_KEY")
+    sk = os.getenv("DEXVERSE_S3_SECRET_KEY")
+    if not (endpoint and ak and sk):
+        return None
+    return boto3.client(
+        "s3", endpoint_url=endpoint,
+        region_name=os.getenv("DEXVERSE_S3_REGION", "us-east-1"),
+        aws_access_key_id=ak, aws_secret_access_key=sk,
+        config=Config(signature_version="s3v4"),
+    )
+
+
+def _build_manifest(task_id: str, task: Dict[str, Any]) -> Dict[str, Any]:
+    endpoint = os.getenv("DEXVERSE_S3_ENDPOINT", "https://minio.dexverse.com")
+    region = os.getenv("DEXVERSE_S3_REGION", "us-east-1")
+    presign_expires = int(os.getenv("DEXVERSE_MOCK_PRESIGN_EXPIRES", "3600"))
+    presigner = _presign_client()
+    expires_at = datetime.now(timezone.utc) + timedelta(hours=12)
+
+    files: List[Dict[str, Any]] = []
+    for item in task.get("files", []):
+        bucket = item["bucket"]
+        object_key = item["objectKey"]
+        relative = item.get("relativePath") or item.get("originalPath") or object_key
+        entry = {
+            "fileId": item.get("fileId", object_key),
+            "uri": f"s3://{bucket}/{object_key}",
+            "bucket": bucket,
+            "objectKey": object_key,
+            "relativePath": relative,
+            "size": item.get("size"),
+            "sha256": item.get("sha256", ""),
+        }
+        # presignedUrl: explicit per-file value, or generated via boto3 if presign is on
+        if item.get("presignedUrl"):
+            entry["presignedUrl"] = item["presignedUrl"]
+        elif presigner is not None:
+            entry["presignedUrl"] = presigner.generate_presigned_url(
+                "get_object",
+                Params={"Bucket": bucket, "Key": object_key},
+                ExpiresIn=presign_expires,
+            )
+        files.append(entry)
+
+    return {
+        "manifestVersion": "1.0",
+        "taskId": task_id,
+        "createdAt": datetime.now(timezone.utc).isoformat(),
+        "expiresAt": expires_at.isoformat(),
+        "storage": {
+            "type": "s3",
+            "endpoint": endpoint,
+            "region": region,
+            "addressingStyle": "path",
+            "credentials": {
+                "type": "dev",
+                "accessKey": os.getenv("DEXVERSE_S3_ACCESS_KEY"),
+                "secretKey": os.getenv("DEXVERSE_S3_SECRET_KEY"),
+                "sessionToken": os.getenv("DEXVERSE_S3_SESSION_TOKEN", ""),
+                "expiresAt": expires_at.isoformat(),
+            },
+        },
+        "download": {
+            "defaultParallel": 16,
+            "verify": "size",
+            "preserveOriginalPath": True,
+        },
+        "files": files,
+    }
+
+
+app = FastAPI(title="DexVerse Mock Backend")
+
+
+@app.get("/api/download-tasks/{task_id}/manifest")
+def get_manifest(task_id: str, authorization: Optional[str] = Header(default=None)) -> Dict[str, Any]:
+    _check_auth(authorization)
+    with _lock:
+        tasks = _load_tasks()
+        if task_id not in tasks:
+            raise HTTPException(status_code=404, detail="Task not found")
+        return _build_manifest(task_id, tasks[task_id])
+
+
+class DevFile(BaseModel):
+    fileId: Optional[str] = None
+    bucket: str
+    objectKey: str
+    relativePath: Optional[str] = None
+    size: Optional[int] = None
+    sha256: Optional[str] = None
+    presignedUrl: Optional[str] = None
+
+
+class DevTaskRequest(BaseModel):
+    files: List[DevFile]
+
+
+@app.post("/api/dev/download-tasks")
+def create_dev_task(
+    req: DevTaskRequest,
+    authorization: Optional[str] = Header(default=None),
+) -> Dict[str, Any]:
+    _check_auth(authorization)
+    task_id = f"dl_dev_{datetime.now(timezone.utc).strftime('%y%m%d%H%M%S')}"
+    with _lock:
+        tasks = _load_tasks()
+        tasks[task_id] = {
+            "files": [f.model_dump(exclude_none=True) for f in req.files],
+        }
+    return {
+        "taskId": task_id,
+        "commandLinux": f"dexverse-dl download --task {task_id} --out /data/dexverse",
+        "commandWindows": f"dexverse-dl download --task {task_id} --out D:\\DexVerse",
+    }
+
+
+@app.get("/healthz")
+def healthz() -> Dict[str, str]:
+    return {"status": "ok"}
+
+
+def build_parser():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="DexVerse mock backend")
+    parser.add_argument("--host", default="127.0.0.1",
+                        help="bind host (default loopback; doc §13.1 item 4)")
+    parser.add_argument("--port", type=int, default=18080)
+    return parser
+
+
+def main(argv=None) -> None:
+    args = build_parser().parse_args(argv)
+
+    import uvicorn
+
+    uvicorn.run(app, host=args.host, port=args.port)
+
+
+if __name__ == "__main__":
+    main()

dexverse_cli-0.0.2.dist-info/METADATA

@@ -0,0 +1,198 @@
+Metadata-Version: 2.4
+Name: dexverse-cli
+Version: 0.0.2
+Summary: DexVerse dataset downloader for S3-compatible robot training datasets
+Author: DexVerse
+License: Apache-2.0
+Keywords: dataset,dexverse,downloader,minio,robotics,s3,s5cmd
+Requires-Python: >=3.9
+Requires-Dist: av>=12
+Requires-Dist: h5py>=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: numpy>=1.26
+Requires-Dist: pillow>=10
+Requires-Dist: pydantic>=2.0
+Requires-Dist: rich>=13.0
+Requires-Dist: tomlkit>=0.12
+Requires-Dist: typer>=0.12
+Provides-Extra: dev
+Requires-Dist: boto3>=1.34; extra == 'dev'
+Requires-Dist: fastapi>=0.110; extra == 'dev'
+Requires-Dist: httpx>=0.27; extra == 'dev'
+Requires-Dist: moto[s3]>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Provides-Extra: mock
+Requires-Dist: fastapi>=0.110; extra == 'mock'
+Requires-Dist: uvicorn>=0.27; extra == 'mock'
+Provides-Extra: s3
+Requires-Dist: boto3>=1.34; extra == 's3'
+Description-Content-Type: text/markdown
+
+# DexVerse Downloader (`dexverse`)
+
+Presigned-URL batch downloader for robot-training datasets.
+
+```text
+manifest -> dexverse-dl -> httpx -> presigned URLs -> local file landing
+```
+
+The primary input is the DexVerse backend's raw presigned-URL JSON. For those
+payloads, `storage_path` is the remote object key and `file_name` is the local
+landing path under `--out`.
+
+Canonical manifests are still supported. Their local paths come from
+`relativePath`, falling back to `originalPath`, then `objectKey`:
+
+```text
+localPath = out / (file.relativePath || file.originalPath || file.objectKey)
+```
+
+## Install
+
+```bash
+pip install "dexverse[s3,mock]"          # boto3 + fastapi mock server
+# optional legacy S3 backend:
+#   install s5cmd (https://github.com/peak/s5cmd) and put it on PATH
+```
+
+Development:
+
+```bash
+uv venv && . .venv/bin/activate          # Windows: .venv\Scripts\activate
+uv pip install -e ".[dev]"
+```
+
+## Quick Start
+
+If your backend issues short-lived presigned URLs per file, save that response
+as `download.json` and run:
+
+```bash
+dexverse-dl download --manifest ./download.json --out D:\DexVerse
+```
+
+No login, no `AWS_*` credentials, no s5cmd, no backend flag, and no TLS flag are
+required for the common internal/self-signed endpoint case. The default backend
+is `http`, and TLS verification is disabled by default for http/presigned
+downloads. Use `--secure` when the endpoint has a valid certificate and you want
+certificate validation.
+
+Raw backend output is accepted directly:
+
+```json
+[{
+  "download_url": "https://host/bucket/datasets/.../20260429_180359.hdf5?X-Amz-Signature=...",
+  "storage_bucket": "dexverse",
+  "storage_path": "datasets/.../20260429_180359.hdf5",
+  "file_name": "20260429_180359.hdf5",
+  "file_size": 14994636,
+  "task_instruction": {
+    "aaa": "111",
+    "aaaa": 222,
+    "ewrwrwe": 1.1111111111111112e+54
+  }
+}]
+```
+
+Result:
+
+```text
+D:\DexVerse\20260429_180359.hdf5
+```
+
+`task_instruction` is optional and must be a JSON object. If multiple manifest
+items resolve to the same local path (for example, repeated `file_name`
+values), the first item wins and later duplicates are ignored.
+
+Include `file_size` (and preferably `sha256`) for large files. The HTTP backend
+also learns the total size from `Content-Length`/`Content-Range`, but manifest
+metadata lets the CLI verify an existing local file before making a request.
+
+Transient timeouts and HTTP 408/425/429/5xx responses are retried five times by
+default. Bytes already received stay in a sibling `<file>.part`; retries and
+later CLI runs request the remaining bytes with HTTP Range, while the final
+destination path only appears after the size check succeeds. The object server
+must support Range requests for efficient resume.
+
+Presigned URLs expire, often after 1 hour. Have the backend regenerate the
+manifest right before each download; expired URLs surface as per-file `HTTP 403`
+errors. After generating a fresh `download.json`, run the same command with the
+same `--out` path to continue from the retained `.part`.
+
+## CLI
+
+```bash
+dexverse-dl login   --server https://dexverse.xxx.com [--token TOKEN]
+dexverse-dl download --manifest ./download.json --out /data/dexverse [--parallel 16] [--secure] [--backend http|python|s5cmd]
+dexverse-dl download --manifest ./download.json --out /data/dexverse [--http-read-timeout 120] [--http-retries 5]
+dexverse-dl download --task dl_xxx --out /data/dexverse [--server URL]
+dexverse-dl verify  --manifest PATH --out /data/dexverse [--deep]
+dexverse-dl repair  --manifest PATH --out /data/dexverse [--backend http|python|s5cmd]
+```
+
+### 还原 HDF5 → 可训练数据
+
+下载完成后，把落盘的 `.hdf5` 还原为原始录制结构（逐帧 JPEG + `pose_record_*.json`
++ `metadata.jsonl`）。如果对应 manifest 带有 `task_instruction`，还会自动生成
+`task_instruction.json`：
+
+```bash
+# 单文件
+dexverse-dl convert -i /data/dexverse/20260429_180359.hdf5 -o /data/dexverse/restored
+# 批量（递归处理目录下所有 *.hdf5，保持目录结构）
+dexverse-dl convert --input-dir /data/dexverse -o /data/dexverse/restored
+```
+
+还原输出形如：
+
+```text
+/data/dexverse/restored/20260429_180359/
+  head/left/000000.jpg ...
+  head/right/000000.jpg ...
+  hand/left/000000.jpg ...
+  hand/right/000000.jpg ...
+  pose_record_20260429_180359.json
+  metadata.jsonl
+  task_instruction.json
+```
+
+`task_instruction.json` 的内容就是 manifest 中的对象本身，不增加外层包装。
+下载器会把转换所需的映射保存在 `.dexverse/task-instructions/`，无需为
+`convert` 增加额外参数。独立来源且没有该隐藏元数据的 HDF5 仍按原行为还原。
+
+该命令依赖 `h5py` / `numpy` / `av` / `pillow`，已列入核心依赖，安装时随包装上。
+
+## Python SDK
+
+```python
+from dexverse import DexVerseClient, download_manifest
+
+client = DexVerseClient(server="https://dexverse.xxx.com", token="xxx")
+local_root = client.download_task(task_id="dl_xxx", out="/data/dexverse", parallel=16)
+
+# or from a local manifest file
+download_manifest(manifest_path="./download.json", out="/data/dexverse")
+```
+
+## Layout
+
+State files live under `<out>/.dexverse/` and never mix into the dataset tree:
+
+```text
+/data/dexverse/
+  .dexverse/task-instructions/
+    20260429_180359.hdf5.json
+  .dexverse/tasks/<taskId>/
+    manifest.json
+    download-state.json
+    download.log
+  20260429_180359.hdf5
+```
+
+## Backends
+
+| backend  | engine            | notes                                   |
+|----------|-------------------|-----------------------------------------|
+| `http`   | httpx             | default; presigned URLs, no creds/login |
+| `python` | boto3             | legacy S3 fallback                      |
+| `s5cmd`  | peak/s5cmd binary | legacy S3 high-throughput backend       |

dexverse_cli-0.0.2.dist-info/RECORD

@@ -0,0 +1,23 @@
+dexverse/__init__.py,sha256=Hf6AqC9YsxJFDCK2BvPpnJ4eKarOPHRb7I6V_n3CzFQ,1365
+dexverse/cli.py,sha256=hWGYEB-tENPXw_CtxJzTJDVwmY7dNHdcmlv30JYd-yw,16802
+dexverse/client.py,sha256=OCchd_x0T20TcegeRIns0SEl_5yb7AQ0POf1D1O3jjc,5073
+dexverse/config.py,sha256=555RAAa5Nyj6E7Ym2m9sU57ndQJU8lanBsJROfTapsU,2368
+dexverse/manifest.py,sha256=uw9V5_L5zf5b8vVa9ihpDmZPYVHepwJkAwiZdjPYz2E,11652
+dexverse/convert/__init__.py,sha256=1O6XCtMzX6ITgXcAQWCntkaJieO2cDwuZzRF7wIDL0g,1001
+dexverse/convert/hdf5_to_raw.py,sha256=GlHGA5C_i5f-hR9B8H_CKCcRfqSm2S20taDq9iXtf0c,19135
+dexverse/convert/image_video_encode.py,sha256=KnYybo3F076HtMUj-3YuwP9kmQED5YBj5DiUxkiOvR4,14283
+dexverse/downloader/__init__.py,sha256=66ATqgVaCd9VDKX7hs_4Bfi_yqLQd3fZjCaoZBuY9Po,723
+dexverse/downloader/base.py,sha256=InH6n264HtmlpArPrlK-F3o2Crd8RDT0KVDPj3a-5pg,3004
+dexverse/downloader/http_backend.py,sha256=ML2Cik4NSgiZQAcuwV1j9nGlthN8sNfekh7PowIczZ0,18666
+dexverse/downloader/manager.py,sha256=ptGWx_f6Ay51FDTodY-jGivWebUazylRtB6VW_eg5k0,14293
+dexverse/downloader/python_s3_backend.py,sha256=-LpvhLn6mer93UP80ugwh21aiKL8r1CWWXPmqwIphaY,5353
+dexverse/downloader/s5cmd_backend.py,sha256=KP28uNRW3DLFGqyEvPlrhh0b8Irm4mCWF5v7sfGKlqs,5897
+dexverse/downloader/state.py,sha256=1iMknv7-SdsAg3rj5_CENKPZACUWv4fS1IPm2Pbiotw,8874
+dexverse/downloader/verify.py,sha256=6M8hZJoUov0Qbpr03B9jUfjObjLqMgE-u_YjWIvqpow,3337
+dexverse/mock/__init__.py,sha256=PfJ4z1g0RHJMeTOAssfiHOIltaktsPIp5J-5pu3EkhA,89
+dexverse/mock/mock_tasks.json,sha256=EyHnM1LOXG5JuP0SYI39R-52LUcXSKx4h34DySg5Iy4,618
+dexverse/mock/server.py,sha256=JLiEEqt8r3DdEbASLn4KIcMQDB1vyBDqezHer3qHBLc,6862
+dexverse_cli-0.0.2.dist-info/METADATA,sha256=0SORmrrWQ07XNPX-Gi-ilU2iiiETAJhPEZPcJy0LLwE,6744
+dexverse_cli-0.0.2.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+dexverse_cli-0.0.2.dist-info/entry_points.txt,sha256=OYxj67T5AD2oVdDxcEOEkSPgy371GA7kWFTMtm74nyw,98
+dexverse_cli-0.0.2.dist-info/RECORD,,

dexverse_cli-0.0.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any

dexverse_cli-0.0.2.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+dexverse-dl = dexverse.cli:app
+dexverse-mock-server = dexverse.mock.server:main