dexverse-cli 0.1.0__tar.gz → 0.1.0.post2__tar.gz

{dexverse_cli-0.1.0 → dexverse_cli-0.1.0.post2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: dexverse-cli
-Version: 0.1.0
+Version: 0.1.0.post2
 Summary: Download, pack, and unpack DexVerse datasets
 Requires-Python: >=3.10
 Requires-Dist: av<17,>=14
@@ -8,6 +8,7 @@ Requires-Dist: h5py<4,>=3.11
 Requires-Dist: httpx<1,>=0.28
 Requires-Dist: numpy<3,>=1.26
 Requires-Dist: pillow<13,>=10
+Requires-Dist: rich<15,>=13.9
 Requires-Dist: typer<1,>=0.16
 Provides-Extra: jetson
 Requires-Dist: pygobject<4,>=3.42; (platform_system == 'Linux') and extra == 'jetson'
@@ -41,7 +42,8 @@ type manifest.json | dexverse-dl download --manifest - --out ./dataset
 
 A manifest is either a JSON file list or an object containing `files`. Each
 file requires `download_url` and `file_name`. Optional fields are
-`storage_path`, `file_size`, `sha256`, and `meta_data`. Downloaded metadata is
+`storage_path`, `file_size`, `sha256`, `meta_data`, and `data_protocol`.
+`data_protocol` is retained in the downloaded metadata sidecar. Downloaded metadata is
 stored in a versioned sidecar and becomes `task_instruction.json` when the
 corresponding HDF5 file is unpacked; only top-level `data_protocol` is removed
 from that algorithm-facing projection.

{dexverse_cli-0.1.0 → dexverse_cli-0.1.0.post2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "dexverse-cli"
-version = "0.1.0"
+version = "0.1.0.post2"
 description = "Download, pack, and unpack DexVerse datasets"
 readme = "src/dexverse_cli/README.md"
 requires-python = ">=3.10"
@@ -14,6 +14,7 @@ dependencies = [
   "httpx>=0.28,<1",
   "numpy>=1.26,<3",
   "pillow>=10,<13",
+  "rich>=13.9,<15",
   "typer>=0.16,<1",
 ]
 

{dexverse_cli-0.1.0 → dexverse_cli-0.1.0.post2}/src/dexverse_cli/README.md

@@ -26,7 +26,8 @@ type manifest.json | dexverse-dl download --manifest - --out ./dataset
 
 A manifest is either a JSON file list or an object containing `files`. Each
 file requires `download_url` and `file_name`. Optional fields are
-`storage_path`, `file_size`, `sha256`, and `meta_data`. Downloaded metadata is
+`storage_path`, `file_size`, `sha256`, `meta_data`, and `data_protocol`.
+`data_protocol` is retained in the downloaded metadata sidecar. Downloaded metadata is
 stored in a versioned sidecar and becomes `task_instruction.json` when the
 corresponding HDF5 file is unpacked; only top-level `data_protocol` is removed
 from that algorithm-facing projection.

dexverse_cli-0.1.0.post2/src/dexverse_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0.post2"

{dexverse_cli-0.1.0 → dexverse_cli-0.1.0.post2}/src/dexverse_cli/cli.py

@@ -1,9 +1,12 @@
 from __future__ import annotations
 
+from collections.abc import Callable
 from pathlib import Path
 from typing import Annotated, Literal
 
 import typer
+from rich.progress import BarColumn, Progress, SpinnerColumn, TaskID, TextColumn, TimeElapsedColumn
+from rich.table import Column
 
 from dexverse_cli.download import download_files
 from dexverse_cli.manifest import ManifestError, read_manifest
@@ -13,6 +16,60 @@ from dexverse_cli.unpack import unpack_batch
 app = typer.Typer(no_args_is_help=True, add_completion=False)
 
 
+def progress_callback(progress: Progress, task_id: TaskID, verb: str) -> Callable[[int, int], None]:
+    def update(completed: int, total: int) -> None:
+        progress.update(task_id, completed=completed, total=total, description=f"{verb} {completed}/{total}")
+
+    return update
+
+
+def unpack_progress_callback(
+    progress: Progress,
+    task_id: TaskID,
+    verb: str,
+) -> Callable[[int, int, int, Path | None, Path | None], None]:
+    def update(
+        completed: int,
+        total: int,
+        current: int,
+        source: Path | None,
+        target: Path | None,
+    ) -> None:
+        description = f"{verb} {current}/{total}" if current else f"{verb} 0/{total}"
+        detail = "Preparing files"
+        if source is not None and target is not None:
+            detail = f"{source.name} → {target}"
+        progress.update(
+            task_id,
+            completed=completed,
+            total=total,
+            description=description,
+            detail=detail,
+        )
+
+    return update
+
+
+def batch_progress(*, show_detail: bool = False) -> Progress:
+    columns = [
+        SpinnerColumn(style="cyan"),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(bar_width=None, style="bright_black", complete_style="cyan"),
+    ]
+    if show_detail:
+        columns.append(
+            TextColumn(
+                "[dim]{task.fields[detail]}",
+                table_column=Column(ratio=2, no_wrap=True, overflow="ellipsis"),
+            )
+        )
+    columns.append(TimeElapsedColumn())
+    return Progress(
+        *columns,
+        transient=True,
+    )
+
+
 @app.command()
 def download(
     manifest: Annotated[str, typer.Option("--manifest", "-m", help="Manifest JSON path or '-'")],
@@ -59,13 +116,16 @@ def pack(
 ) -> None:
     """Pack one or more raw dataset sessions into Hdf5SchemaV1 files."""
     try:
-        report = pack_batch(
-            input_path,
-            output,
-            output_format=output_format,
-            skip_existing=skip_existing,
-            workers=workers,
-        )
+        with batch_progress() as progress:
+            task_id = progress.add_task("Finding sessions", total=None)
+            report = pack_batch(
+                input_path,
+                output,
+                output_format=output_format,
+                skip_existing=skip_existing,
+                workers=workers,
+                on_progress=progress_callback(progress, task_id, "Packing"),
+            )
     except (OSError, ValueError) as exc:
         typer.echo(f"error: {exc}", err=True)
         raise typer.Exit(code=1) from exc
@@ -117,7 +177,9 @@ def run_unpack(
     if not source.exists():
         typer.echo(f"error: input path does not exist: {source}", err=True)
         raise typer.Exit(code=1)
-    report = unpack_batch(source, output)
+    with batch_progress(show_detail=True) as progress:
+        task_id = progress.add_task("Finding sessions", total=None, detail="Scanning input")
+        report = unpack_batch(source, output, on_progress=unpack_progress_callback(progress, task_id, verb.title()))
     typer.echo(f"{verb}={report.unpacked} frames={report.total_frames} failed={len(report.failures)}")
     for failure in report.failures:
         typer.echo(f"error: {failure}", err=True)

{dexverse_cli-0.1.0 → dexverse_cli-0.1.0.post2}/src/dexverse_cli/manifest.py

@@ -17,6 +17,7 @@ _ALLOWED_FIELDS = frozenset(
         "file_size",
         "sha256",
         "meta_data",
+        "data_protocol",
     }
 )
 _DRIVE_PREFIX = re.compile(r"^[A-Za-z]:")
@@ -116,12 +117,19 @@ def optional_sha256(value: object, *, index: int) -> str | None:
 
 
 def metadata(item: dict[str, Any], *, index: int) -> dict[str, Any] | None:
-    if "meta_data" not in item:
-        return None
-    value = item["meta_data"]
-    if not isinstance(value, dict):
+    value = item.get("meta_data") if "meta_data" in item else None
+    if "meta_data" in item and not isinstance(value, dict):
         raise ManifestError(f"files[{index}].meta_data must be a JSON object")
-    return value
+    merged = dict(value) if value is not None else {}
+    if "data_protocol" not in item:
+        return merged or None
+    data_protocol = item["data_protocol"]
+    if not isinstance(data_protocol, str) or not data_protocol or data_protocol != data_protocol.strip():
+        raise ManifestError(f"files[{index}].data_protocol must be a non-empty string")
+    if "data_protocol" in merged and merged["data_protocol"] != data_protocol:
+        raise ManifestError(f"files[{index}] has conflicting data_protocol values")
+    merged["data_protocol"] = data_protocol
+    return merged
 
 
 def claim_parts(file_name: str) -> tuple[str, ...]:

{dexverse_cli-0.1.0 → dexverse_cli-0.1.0.post2}/src/dexverse_cli/pack.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import io
 import os
+from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from pathlib import Path, PurePosixPath
@@ -16,6 +17,7 @@ from dexverse_cli.raw_schema import PackValidationPolicyV1, RawSessionV1, Valida
 from dexverse_cli.transaction import publish_file
 
 PackFormat = Literal["jpeg", "video"]
+ProgressCallback = Callable[[int, int], None]
 
 
 @dataclass(frozen=True, slots=True)
@@ -52,10 +54,13 @@ def pack_batch(
     output_format: PackFormat = "jpeg",
     skip_existing: bool = False,
     workers: int | None = None,
+    on_progress: ProgressCallback | None = None,
 ) -> PackReport:
     sources = discover_raw_sessions(input_path)
     if not sources:
         return PackReport(0, 0, (f"no raw sessions found: {input_path}",))
+    if on_progress is not None:
+        on_progress(0, len(sources))
     if workers is None:
         workers = min(len(sources), os.cpu_count() or 1, 32)
     if workers < 1 or workers > 32:
@@ -74,12 +79,17 @@ def pack_batch(
         except Exception as exc:
             return "failed", f"{source}: {exc}"
 
-    results: tuple[tuple[str, str | None], ...]
+    results: list[tuple[str, str | None]] = []
     if len(sources) == 1:
-        results = (process(sources[0]),)
+        results.append(process(sources[0]))
+        if on_progress is not None:
+            on_progress(1, 1)
     else:
         with ThreadPoolExecutor(max_workers=workers) as executor:
-            results = tuple(executor.map(process, sources))
+            for completed, result in enumerate(executor.map(process, sources), start=1):
+                results.append(result)
+                if on_progress is not None:
+                    on_progress(completed, len(sources))
     return PackReport(
         packed=sum(status == "packed" for status, _message in results),
         skipped=sum(status == "skipped" for status, _message in results),

{dexverse_cli-0.1.0 → dexverse_cli-0.1.0.post2}/src/dexverse_cli/unpack.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import io
 import json
+from collections.abc import Callable
 from dataclasses import dataclass
 from pathlib import Path, PurePosixPath
 from typing import Any
@@ -15,6 +16,8 @@ from dexverse_cli.state import file_metadata_path, load_file_metadata
 from dexverse_cli.transaction import publish_directory
 from dexverse_cli.unversioned_hdf5 import restore_unversioned_hdf5
 
+ProgressCallback = Callable[[int, int, int, Path | None, Path | None], None]
+
 
 @dataclass(frozen=True, slots=True)
 class UnpackReport:
@@ -35,23 +38,35 @@ def discover_hdf5(input_path: Path) -> tuple[Path, ...]:
     return ()
 
 
-def unpack_batch(input_path: Path, output: Path) -> UnpackReport:
+def unpack_batch(
+    input_path: Path,
+    output: Path,
+    *,
+    on_progress: ProgressCallback | None = None,
+) -> UnpackReport:
     sources = discover_hdf5(input_path)
     if not sources:
         return UnpackReport(0, 0, (f"no supported HDF5 files found: {input_path}",))
+    if on_progress is not None:
+        on_progress(0, len(sources), 0, None, None)
     unpacked = 0
     total_frames = 0
     failures: list[str] = []
-    for source in sources:
+    for completed, source in enumerate(sources, start=1):
         target = (
             output / source.relative_to(input_path).with_suffix("") if input_path.is_dir() else output / source.stem
         )
+        if on_progress is not None:
+            on_progress(completed - 1, len(sources), completed, source, target)
         try:
             frames = unpack_transactionally(source, target)
             unpacked += 1
             total_frames += frames
         except Exception as exc:
             failures.append(f"{source}: {exc}")
+        finally:
+            if on_progress is not None:
+                on_progress(completed, len(sources), completed, source, target)
     return UnpackReport(unpacked, total_frames, tuple(failures))
 
 

{dexverse_cli-0.1.0 → dexverse_cli-0.1.0.post2}/uv.lock

@@ -108,7 +108,7 @@ wheels = [
 
 [[package]]
 name = "dexverse-cli"
-version = "0.1.0"
+version = "0.1.0.post2"
 source = { editable = "." }
 dependencies = [
     { name = "av" },
@@ -118,6 +118,7 @@ dependencies = [
     { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" },
     { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
     { name = "pillow" },
+    { name = "rich" },
     { name = "typer" },
 ]
 
@@ -141,6 +142,7 @@ requires-dist = [
     { name = "numpy", specifier = ">=1.26,<3" },
     { name = "pillow", specifier = ">=10,<13" },
     { name = "pygobject", marker = "sys_platform == 'linux' and extra == 'jetson'", specifier = ">=3.42,<4" },
+    { name = "rich", specifier = ">=13.9,<15" },
     { name = "typer", specifier = ">=0.16,<1" },
 ]
 provides-extras = ["jetson"]
@@ -886,15 +888,15 @@ wheels = [
 
 [[package]]
 name = "rich"
-version = "15.0.0"
+version = "14.3.4"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "markdown-it-py" },
     { name = "pygments" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/e9/67/cae617f1351490c25a4b8ac3b8b63a4dda609295d8222bad12242dfdc629/rich-14.3.4.tar.gz", hash = "sha256:817e02727f2b25b40ef56f5aa2217f400c8489f79ca8f46ea2b70dd5e14558a9", size = 230524, upload-time = "2026-04-11T02:57:45.419Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/76/6d163cfac87b632216f71879e6b2cf17163f773ff59c00b5ff4900a80fa3/rich-14.3.4-py3-none-any.whl", hash = "sha256:07e7adb4690f68864777b1450859253bed81a99a31ac321ac1817b2313558952", size = 310480, upload-time = "2026-04-11T02:57:47.484Z" },
 ]
 
 [[package]]

dexverse_cli-0.1.0/src/dexverse_cli/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.1.0"