dexverse-cli 0.0.4.post1__tar.gz → 0.1.0__tar.gz

dexverse_cli-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+.venv/
+.venv-smoke/
+.venv-smoke-*/
+.smoke-work/
+.artifact-inspect/
+.manifest/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+__pycache__/
+*.py[cod]
+dist/

dexverse_cli-0.1.0/.python-version

@@ -0,0 +1 @@
+3.10

dexverse_cli-0.1.0/PKG-INFO

@@ -0,0 +1,112 @@
+Metadata-Version: 2.5
+Name: dexverse-cli
+Version: 0.1.0
+Summary: Download, pack, and unpack DexVerse datasets
+Requires-Python: >=3.10
+Requires-Dist: av<17,>=14
+Requires-Dist: h5py<4,>=3.11
+Requires-Dist: httpx<1,>=0.28
+Requires-Dist: numpy<3,>=1.26
+Requires-Dist: pillow<13,>=10
+Requires-Dist: typer<1,>=0.16
+Provides-Extra: jetson
+Requires-Dist: pygobject<4,>=3.42; (platform_system == 'Linux') and extra == 'jetson'
+Description-Content-Type: text/markdown
+
+# dexverse-cli
+
+`dexverse-cli` downloads DexVerse dataset manifests over HTTP(S), packs strict
+raw sessions into versioned HDF5 files, and unpacks HDF5 sessions. It requires
+Python 3.10 or newer.
+
+## Install
+
+```console
+pip install dexverse-cli
+```
+
+## Download
+
+Read a manifest from a file:
+
+```console
+dexverse-dl download --manifest manifest.json --out ./dataset
+```
+
+Or pipe a short-lived manifest directly to stdin:
+
+```console
+type manifest.json | dexverse-dl download --manifest - --out ./dataset
+```
+
+A manifest is either a JSON file list or an object containing `files`. Each
+file requires `download_url` and `file_name`. Optional fields are
+`storage_path`, `file_size`, `sha256`, and `meta_data`. Downloaded metadata is
+stored in a versioned sidecar and becomes `task_instruction.json` when the
+corresponding HDF5 file is unpacked; only top-level `data_protocol` is removed
+from that algorithm-facing projection.
+
+Downloads use bounded parallelism, retries, and compatible partial-file resume.
+When `sha256` is present it is always checked before the final file is replaced.
+TLS certificate verification is disabled by default for internal endpoints;
+pass `--secure` to enable certificate verification.
+
+## Pack
+
+Pack one raw session, or the immediate session directories under a parent:
+
+```console
+dexverse-dl pack --input ./raw/session --output ./packed --format jpeg
+```
+
+JPEG is the platform-independent default. Packing validates the complete raw
+inventory, every image, all camera stream relationships, and pose data before
+transactionally replacing an output. Camera and pose counts may differ; pose is
+linearly aligned to the deterministic reference camera timeline.
+
+Jetson MP4 packing is optional:
+
+```console
+pip install "dexverse-cli[jetson]"
+dexverse-dl pack --input ./raw/session --output ./packed --format video
+```
+
+The Python extra does not install native Jetson GStreamer components. Video
+packing reports an explicit error unless the required hardware pipeline is
+available. JPEG packing, download, help, and unpack do not import GStreamer.
+
+## Unpack
+
+Unpack one HDF5 file:
+
+```console
+dexverse-dl unpack --input ./packed/session.hdf5 --output ./restored
+```
+
+Or recursively unpack `*.hdf5` files while preserving their relative paths:
+
+```console
+dexverse-dl unpack --input-dir ./packed --output ./restored
+```
+
+Unpacking validates the complete versioned schema before transactionally
+publishing a restored session. It also retains the supported unversioned HDF5
+read path.
+
+## Convert
+
+Convert one supported HDF5 file:
+
+```console
+dexverse-dl convert --input ./dataset/session.hdf5 --output ./restored
+```
+
+Or recursively convert `*.hdf5` files while preserving their relative paths:
+
+```console
+dexverse-dl convert --input-dir ./dataset --output ./restored
+```
+
+`convert` is the compatibility alias for `unpack`; it retains the same options,
+target mapping, legacy HDF5 support, exit codes, and `converted=...` summary.
+Batches continue after an item failure but exit with status 1 if any item fails.

dexverse_cli-0.1.0/pyproject.toml

@@ -0,0 +1,67 @@
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[project]
+name = "dexverse-cli"
+version = "0.1.0"
+description = "Download, pack, and unpack DexVerse datasets"
+readme = "src/dexverse_cli/README.md"
+requires-python = ">=3.10"
+dependencies = [
+  "av>=14,<17",
+  "h5py>=3.11,<4",
+  "httpx>=0.28,<1",
+  "numpy>=1.26,<3",
+  "pillow>=10,<13",
+  "typer>=0.16,<1",
+]
+
+[project.optional-dependencies]
+jetson = [
+  "PyGObject>=3.42,<4; platform_system == 'Linux'",
+]
+
+[project.scripts]
+dexverse-dl = "dexverse_cli.cli:app"
+
+[dependency-groups]
+dev = [
+  "mypy>=1.15,<2",
+  "pytest>=8.3,<10",
+  "ruff>=0.11,<1",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dexverse_cli"]
+
+[tool.hatch.build.targets.sdist]
+include = [
+  "/pyproject.toml",
+  "/uv.lock",
+  "/.python-version",
+  "/src/dexverse_cli",
+]
+
+[tool.pytest.ini_options]
+addopts = "-ra"
+testpaths = ["tests"]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 120
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM", "RUF", "TID252"]
+
+[tool.ruff.lint.flake8-tidy-imports]
+ban-relative-imports = "all"
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**/*.py" = ["TID252"]
+
+[tool.mypy]
+python_version = "3.10"
+strict = true
+warn_unreachable = true
+ignore_missing_imports = true

dexverse_cli-0.1.0/src/dexverse_cli/README.md

@@ -0,0 +1,97 @@
+# dexverse-cli
+
+`dexverse-cli` downloads DexVerse dataset manifests over HTTP(S), packs strict
+raw sessions into versioned HDF5 files, and unpacks HDF5 sessions. It requires
+Python 3.10 or newer.
+
+## Install
+
+```console
+pip install dexverse-cli
+```
+
+## Download
+
+Read a manifest from a file:
+
+```console
+dexverse-dl download --manifest manifest.json --out ./dataset
+```
+
+Or pipe a short-lived manifest directly to stdin:
+
+```console
+type manifest.json | dexverse-dl download --manifest - --out ./dataset
+```
+
+A manifest is either a JSON file list or an object containing `files`. Each
+file requires `download_url` and `file_name`. Optional fields are
+`storage_path`, `file_size`, `sha256`, and `meta_data`. Downloaded metadata is
+stored in a versioned sidecar and becomes `task_instruction.json` when the
+corresponding HDF5 file is unpacked; only top-level `data_protocol` is removed
+from that algorithm-facing projection.
+
+Downloads use bounded parallelism, retries, and compatible partial-file resume.
+When `sha256` is present it is always checked before the final file is replaced.
+TLS certificate verification is disabled by default for internal endpoints;
+pass `--secure` to enable certificate verification.
+
+## Pack
+
+Pack one raw session, or the immediate session directories under a parent:
+
+```console
+dexverse-dl pack --input ./raw/session --output ./packed --format jpeg
+```
+
+JPEG is the platform-independent default. Packing validates the complete raw
+inventory, every image, all camera stream relationships, and pose data before
+transactionally replacing an output. Camera and pose counts may differ; pose is
+linearly aligned to the deterministic reference camera timeline.
+
+Jetson MP4 packing is optional:
+
+```console
+pip install "dexverse-cli[jetson]"
+dexverse-dl pack --input ./raw/session --output ./packed --format video
+```
+
+The Python extra does not install native Jetson GStreamer components. Video
+packing reports an explicit error unless the required hardware pipeline is
+available. JPEG packing, download, help, and unpack do not import GStreamer.
+
+## Unpack
+
+Unpack one HDF5 file:
+
+```console
+dexverse-dl unpack --input ./packed/session.hdf5 --output ./restored
+```
+
+Or recursively unpack `*.hdf5` files while preserving their relative paths:
+
+```console
+dexverse-dl unpack --input-dir ./packed --output ./restored
+```
+
+Unpacking validates the complete versioned schema before transactionally
+publishing a restored session. It also retains the supported unversioned HDF5
+read path.
+
+## Convert
+
+Convert one supported HDF5 file:
+
+```console
+dexverse-dl convert --input ./dataset/session.hdf5 --output ./restored
+```
+
+Or recursively convert `*.hdf5` files while preserving their relative paths:
+
+```console
+dexverse-dl convert --input-dir ./dataset --output ./restored
+```
+
+`convert` is the compatibility alias for `unpack`; it retains the same options,
+target mapping, legacy HDF5 support, exit codes, and `converted=...` summary.
+Batches continue after an item failure but exit with status 1 if any item fails.

dexverse_cli-0.1.0/src/dexverse_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

dexverse_cli-0.1.0/src/dexverse_cli/alignment.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+import numpy.typing as npt
+
+from dexverse_cli.raw_schema import RawSessionV1
+
+
+@dataclass(frozen=True, slots=True)
+class AlignedPoseV1:
+    data: npt.NDArray[np.float32]
+    timestamps: npt.NDArray[np.uint64]
+    columns: tuple[str, ...]
+
+
+class PackAlignmentPolicyV1:
+    """Linearly align pose values to the deterministic reference camera timeline."""
+
+    name = "linear_pose_to_reference_camera_v1"
+
+    def align(self, session: RawSessionV1) -> AlignedPoseV1:
+        target = np.asarray(
+            [record.timestamp for record in session.records_by_camera[session.reference_camera]],
+            dtype=np.float64,
+        )
+        timestamps = np.asarray(
+            [record.timestamp_ns for record in session.records_by_camera[session.reference_camera]],
+            dtype=np.uint64,
+        )
+        columns = session.pose_columns
+        result = np.empty((target.size, len(columns)), dtype=np.float32)
+        source_times = np.asarray([frame.timestamp for frame in session.pose_frames], dtype=np.float64)
+        for column_index, column in enumerate(columns):
+            values = np.asarray(
+                [frame.data[column] for frame in session.pose_frames],
+                dtype=np.float64,
+            )
+            if values.size == 1:
+                aligned = np.full(target.shape, values[0], dtype=np.float64)
+            else:
+                aligned = np.interp(target, source_times, values)
+            result[:, column_index] = aligned.astype(np.float32)
+        return AlignedPoseV1(result, timestamps, columns)

dexverse_cli-0.1.0/src/dexverse_cli/cli.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated, Literal
+
+import typer
+
+from dexverse_cli.download import download_files
+from dexverse_cli.manifest import ManifestError, read_manifest
+from dexverse_cli.pack import pack_batch
+from dexverse_cli.unpack import unpack_batch
+
+app = typer.Typer(no_args_is_help=True, add_completion=False)
+
+
+@app.command()
+def download(
+    manifest: Annotated[str, typer.Option("--manifest", "-m", help="Manifest JSON path or '-'")],
+    out: Annotated[Path, typer.Option("--out", "-o", help="Local download root")],
+    parallel: Annotated[int, typer.Option("--parallel", "-p", min=1, max=32)] = 4,
+    http_read_timeout: Annotated[float, typer.Option("--http-read-timeout", min=0.1)] = 120.0,
+    http_retries: Annotated[int, typer.Option("--http-retries", min=0)] = 5,
+    secure: Annotated[bool, typer.Option("--secure", help="Verify server TLS certificates")] = False,
+) -> None:
+    """Download all files in an HTTP(S) manifest."""
+    try:
+        files = read_manifest(manifest)
+        if not secure:
+            typer.echo(
+                "warning: TLS certificate verification is disabled; pass --secure to enable it",
+                err=True,
+            )
+        report = download_files(
+            files,
+            out,
+            parallel=parallel,
+            secure=secure,
+            read_timeout=http_read_timeout,
+            max_retries=http_retries,
+        )
+    except (ManifestError, OSError) as exc:
+        typer.echo(f"error: {exc}", err=True)
+        raise typer.Exit(code=1) from exc
+
+    typer.echo(f"downloaded={report.downloaded} skipped={report.skipped} failed={len(report.failures)}")
+    for failure in report.failures:
+        typer.echo(f"error: {failure}", err=True)
+    if not report.ok:
+        raise typer.Exit(code=1)
+
+
+@app.command()
+def pack(
+    input_path: Annotated[Path, typer.Option("--input", "-i")],
+    output: Annotated[Path, typer.Option("--output", "-o")],
+    output_format: Annotated[Literal["jpeg", "video"], typer.Option("--format")] = "jpeg",
+    skip_existing: Annotated[bool, typer.Option("--skip-existing")] = False,
+    workers: Annotated[int | None, typer.Option("--workers", min=1, max=32)] = None,
+) -> None:
+    """Pack one or more raw dataset sessions into Hdf5SchemaV1 files."""
+    try:
+        report = pack_batch(
+            input_path,
+            output,
+            output_format=output_format,
+            skip_existing=skip_existing,
+            workers=workers,
+        )
+    except (OSError, ValueError) as exc:
+        typer.echo(f"error: {exc}", err=True)
+        raise typer.Exit(code=1) from exc
+    typer.echo(f"packed={report.packed} skipped={report.skipped} failed={len(report.failures)}")
+    for failure in report.failures:
+        typer.echo(f"error: {failure}", err=True)
+    if not report.ok:
+        raise typer.Exit(code=1)
+
+
+@app.command()
+def unpack(
+    output: Annotated[Path, typer.Option("--output", "-o")],
+    input_file: Annotated[Path | None, typer.Option("--input", "-i")] = None,
+    input_dir: Annotated[Path | None, typer.Option("--input-dir")] = None,
+    strategy: Annotated[str, typer.Option("--strategy")] = "default",
+) -> None:
+    """Unpack HDF5 sessions to the restored raw dataset layout."""
+    run_unpack(output, input_file, input_dir, strategy, verb="unpacked")
+
+
+@app.command()
+def convert(
+    output: Annotated[Path, typer.Option("--output", "-o")],
+    input_file: Annotated[Path | None, typer.Option("--input", "-i")] = None,
+    input_dir: Annotated[Path | None, typer.Option("--input-dir")] = None,
+    strategy: Annotated[str, typer.Option("--strategy")] = "default",
+) -> None:
+    """Convert HDF5 sessions to the restored dataset layout."""
+    run_unpack(output, input_file, input_dir, strategy, verb="converted")
+
+
+def run_unpack(
+    output: Path,
+    input_file: Path | None,
+    input_dir: Path | None,
+    strategy: str,
+    *,
+    verb: Literal["unpacked", "converted"],
+) -> None:
+    if strategy != "default":
+        typer.echo("error: --strategy must be 'default'", err=True)
+        raise typer.Exit(code=2)
+    if (input_file is None) == (input_dir is None):
+        typer.echo("error: pass exactly one of --input or --input-dir", err=True)
+        raise typer.Exit(code=2)
+    source = input_file if input_file is not None else input_dir
+    assert source is not None
+    if not source.exists():
+        typer.echo(f"error: input path does not exist: {source}", err=True)
+        raise typer.Exit(code=1)
+    report = unpack_batch(source, output)
+    typer.echo(f"{verb}={report.unpacked} frames={report.total_frames} failed={len(report.failures)}")
+    for failure in report.failures:
+        typer.echo(f"error: {failure}", err=True)
+    if not report.ok:
+        raise typer.Exit(code=1)