dexverse-cli 0.1.3.post1__tar.gz → 0.1.4__tar.gz

{dexverse_cli-0.1.3.post1 → dexverse_cli-0.1.4}/.gitignore

@@ -1,12 +1,12 @@
-.venv/
-.venv-smoke/
-.venv-smoke-*/
-.smoke-work/
-.artifact-inspect/
-.manifest/
-.pytest_cache/
-.mypy_cache/
-.ruff_cache/
-__pycache__/
-*.py[cod]
-dist/
+.venv/
+.venv-smoke/
+.venv-smoke-*/
+.smoke-work/
+.artifact-inspect/
+.manifest/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+__pycache__/
+*.py[cod]
+dist/

{dexverse_cli-0.1.3.post1 → dexverse_cli-0.1.4}/.python-version

@@ -1 +1 @@
-3.10
+3.10

dexverse_cli-0.1.4/PKG-INFO

@@ -0,0 +1,301 @@
+Metadata-Version: 2.5
+Name: dexverse-cli
+Version: 0.1.4
+Summary: Download DexVerse dataset manifests and unpack HDF5 sessions
+Requires-Python: >=3.10
+Requires-Dist: av<17,>=14
+Requires-Dist: h5py<4,>=3.11
+Requires-Dist: httpx<1,>=0.28
+Requires-Dist: numpy<3,>=1.26
+Requires-Dist: packaging<27,>=24
+Requires-Dist: pillow<13,>=10
+Requires-Dist: rich<15,>=13.9
+Requires-Dist: typer<1,>=0.16
+Description-Content-Type: text/markdown
+
+# dexverse-cli
+
+dexverse-cli downloads DexVerse dataset manifests over HTTP(S) and unpacks
+HDF5 sessions back to the restored dataset layout. It requires
+Python 3.10 or newer.
+
+
+## Install
+
+```console
+pip install dexverse-cli
+```
+
+## Automatic updates
+
+Every business command (`download`, `unpack`, `convert`) first asks whether a
+newer `dexverse-cli` is published. Public PyPI is asked first, because that is
+where a new release appears immediately; when PyPI cannot be reached the check
+falls back to the source this machine's pip is configured with (`pip.conf`,
+`PIP_INDEX_URL`, mirrors, proxies). The check is bounded to about two seconds.
+
+- Nothing new, or nothing to ask: the command runs as usual.
+- A newer release exists: the CLI upgrades itself in place with
+  `python -m pip install --upgrade dexverse-cli` from the source that answered,
+  and then exits with status **20**. Run the same command again; the rerun uses
+  the new version.
+- The check times out, pip is missing, no source answers, or the upgrade fails:
+  one line is printed and the requested command still runs.
+
+The switch is on by default. Turn the check off for one command, or permanently:
+
+```console
+dexverse-dl download --manifest manifest.json --out ./dataset --no-update-check
+set DEXVERSE_CLI_UPDATE=off        ::  Windows (export ... on Linux and macOS)
+```
+
+To pin one command — or a whole machine — to a specific source instead:
+
+```console
+dexverse-dl download --manifest manifest.json --out ./dataset --update-index-url https://mirror.internal/simple
+set DEXVERSE_CLI_UPDATE_INDEX_URL=https://mirror.internal/simple
+set DEXVERSE_CLI_UPDATE_EXTRA_INDEX_URL=https://mirror.internal/simple
+set DEXVERSE_CLI_UPDATE_TRUSTED_HOST=mirror.internal
+```
+
+The flag is shorthand for `DEXVERSE_CLI_UPDATE_INDEX_URL`. Naming any of these
+replaces both defaults: pip is asked only there, which is also how to skip the
+PyPI attempt on a network that cannot reach it.
+
+`dexverse-dl --update-status` prints the recorded observation as JSON without any
+network access or change: whether the switch is on, which source a check would
+use (`update_source`, either `pip` or the index you named), how this interpreter
+was installed (`install_mode`), and the last upgrade this environment performed,
+if any. A command that changes nothing records nothing, so it leaves no directory
+behind. `dexverse-dl --version` never checks. Pre-releases are ignored unless you
+set `DEXVERSE_CLI_UPDATE_PRERELEASE=1`, and a working copy (`pip install -e .`,
+`install_mode: source`) is never checked or upgraded automatically.
+
+## Download
+
+Read a manifest from a file:
+
+```console
+dexverse-dl download --manifest manifest.json --out ./dataset
+```
+
+Or pipe a short-lived manifest directly to stdin:
+
+```console
+type manifest.json | dexverse-dl download --manifest - --out ./dataset
+```
+
+A manifest is either a JSON file list or an object containing `files`. Each
+file requires `download_url` and `file_name`. Optional fields are
+`storage_path`, `file_size`, `sha256`, `hash`, `hash_type`, `meta_data`, and
+`data_protocol`. Unknown top-level keys next to `files` (for example the
+`query_conditions` / `export_info` blocks written by the web batch export) are
+ignored. `data_protocol` is retained in the downloaded metadata sidecar.
+Downloaded metadata is stored in a versioned sidecar and becomes
+`task_instruction.json` when the corresponding HDF5 file is unpacked; only
+top-level `data_protocol` is removed from that algorithm-facing projection.
+
+### Digests
+
+A manifest may declare one digest per file. `hash` and `hash_type` are a pair
+and must appear together, so the tool never guesses an algorithm; the legacy
+`sha256` field is the one exception, since its name already carries the type.
+Three types are accepted:
+
+| `hash_type` | Meaning | Verification cost |
+| --- | --- | --- |
+| `sha256` | Whole-file SHA-256 | Reads the whole file after download |
+| `md5` | Whole-file MD5, for older records | Reads the whole file after download |
+| `sample_sha256_v1` | Bounded sample over the head, middle, and tail 1 MiB blocks, framed with the file size | Reads at most 3 MiB per file |
+
+`sample_sha256_v1` matches the digest the uploader and the web console already
+produce, so their recorded value can be reused directly. It is a deduplication
+fingerprint, not a full checksum: a change outside the sampled windows cannot be
+detected. It does catch truncation and any change inside a sampled window.
+`md5` is fine for detecting transfer corruption but is weak against a deliberate
+collision, so it is not a security boundary.
+
+Digests are **not** checked unless you ask for it, because hashing large files
+costs real time:
+
+```console
+dexverse-dl download --manifest manifest.json --out ./dataset --verify
+```
+
+Without `--verify`, only `file_size` is enforced (when the manifest declares
+it), and the download state records no digest — so a later `--verify` run still
+rechecks the file instead of trusting an unchecked claim. With `--verify`, a
+mismatch removes the partial and fails that file.
+
+The download command renders one progress display per batch: a batch line
+(percent, file counter, transferred bytes, rate, remaining time, elapsed time)
+and a line naming the current file. A retry prints a single persistent warning
+above the display and updates the same progress line, so no second progress bar
+is ever added:
+
+```console
+Download   ━━━━━━━━━━━━━━───────────────────────   38%  13/33  88.0 GB/228.3 GB  19.4 MB/s  1:15:47 < 2:00:49
+downloading: piper_v02_0920_raw/episode_000007.hdf5
+warning: retrying piper_v02_0920_raw/episode_000012.hdf5 (attempt 2/5) in 2.0s: transient HTTP status 503
+Download   ━━━━━━━━━━━━━━────────────────────────   38%  13/33  1 failed  88.0 GB/228.3 GB  19.4 MB/s  1:15:47 < 2:00:49
+retrying: piper_v02_0920_raw/episode_000012.hdf5 (retry 2/5) - transient HTTP status 503; next attempt in 2.0s
+```
+
+The bar is a filled rail drawn with rich's own glyphs: `━` for a filled cell and
+`─` for the rest. There is no frame around it, and no mark drawn inside the empty
+part either: unlike a space-filled bar the rail delimits itself, which is why
+`pip`, `rich`, and `tqdm` leave the brackets out. rich builds its rail from one
+glyph and shows the split with colour, which is why rich can afford half cells;
+here the two weights carry the split, and that rules half cells out. A half cell of
+fill is heavy on one side of its cell and blank on the other, while a light rail
+can only start at the next boundary, so the junction gains and loses half a cell
+as the fill crosses each one — a notch that blinks. The fill therefore ends on
+cell boundaries only and meets the rail directly, so every frame has one junction
+shape and only its position moves; the percentage and byte counters carry the finer
+resolution. Holding a permanent half-cell gap open at the junction was tried as
+well, and reads worse than meeting flush, so there is none. The fill is drawn in
+the line's own colour and only the rest of the rail steps back to a dim shade, so
+the bar never competes with the numbers. Because the empty part differs by glyph
+and not by colour alone, progress stays readable in a piped log and under
+`NO_COLOR`. Counters and byte totals carry no words — `13/33` and
+`88.0 GB/228.3 GB` are self-describing, as they are in every common progress
+line — and the two clocks are paired the way `tqdm` pairs them, elapsed on the
+left of `<` and remaining on the right, so neither needs a label and neither can
+be mistaken for the other. The right side stays `--:--` while the total is
+incomplete, since only a complete total yields an estimate.
+
+The rail keeps one length for the whole batch: the first frame fixes it, so a
+field that appears or grows mid-run (a failure count, a byte total crossing into
+the next unit, the elapsed clock) cannot take cells from it. A shorter bar would
+re-render a shorter fill and read as progress going backwards. That first frame
+also holds room for the failure count before anything has failed, so the first
+failure cannot push the remaining time off the end of the line. Only a terminal
+resize measures the rail again, and on a narrow terminal the tail is clipped
+before the rail is. The rail is up to 40 cells — long enough that the first one
+fills at about 2.5%, since a bar that only starts moving at 4% is doing less to
+calm the wait than it could — and it takes whatever fewer columns a narrow
+terminal leaves over.
+
+The live region is rebuilt on rich's refresh tick as well as whenever new state
+arrives, so the elapsed clock keeps counting while one file is being worked on.
+Without that tick the clock would freeze between callbacks and then jump by the
+length of the file, which reads as a stall.
+
+Percent, byte counters, and the bar follow transferred bytes; the file counter
+advances when a file is published or skipped. When the manifest omits
+`file_size` for any entry, the total is shown as a lower bound (`~6.3 GB`) that
+grows as each transfer reports its real size, and the bar stays an empty rail
+rather than filling against a bound that is still growing; percent and remaining
+time are withheld until the total is complete instead of being guessed from a
+partial sum. Pass `--probe-sizes` to read those sizes from the server first (one
+small ranged request per entry that lacks one), so the total, percentage, and
+remaining time are correct before the first byte moves:
+
+```console
+dexverse-dl download --manifest manifest.json --out ./dataset --probe-sizes
+```
+
+The probe is off by default because a complete manifest needs no extra requests.
+It costs one `GET` with `Range: bytes=0-0` per unknown entry and tolerates a
+server that ignores the range. Piped or logged output (no terminal) prints the
+same line periodically instead of repainting in place, so redirecting to a file
+stays readable.
+
+Pass `--no-progress` to suppress the live display, the per-file line, and retry
+warnings while keeping the final summary:
+
+```console
+dexverse-dl download --manifest manifest.json --out ./dataset --no-progress
+```
+
+
+Downloads use bounded parallelism, retries, and compatible partial-file resume.
+Within a manifest, the first occurrence of a target path keeps its name; later
+occurrences receive `-1`, `-2`, etc. before the final extension (for example,
+`episode1.hdf5`, `episode1-1.hdf5`, `episode1-2.hdf5`). Numbering skips names and
+paths already claimed by other entries, including partial files and metadata.
+Different relative directories remain independent. Names are assigned in manifest
+order, with Windows case-insensitive matching, before any downloads start.
+Rerunning the same manifest preserves the assigned names for skipping and resume;
+existing files on disk do not trigger new suffixes. This is not cross-batch name
+allocation: changing the manifest entries or their order can change the mapping.
+When a digest is declared it is checked only under `--verify`; without it a file
+is accepted on its declared `file_size` alone. TLS certificate verification is
+disabled by default for internal endpoints; pass `--secure` to enable
+certificate verification.
+
+## Unpack
+
+Unpack one HDF5 file to the restored raw dataset layout:
+
+```console
+dexverse-dl unpack --input ./packed/session.hdf5 --output ./restored
+```
+
+Or unpack every `*.hdf5` under a directory, preserving relative paths:
+
+```console
+dexverse-dl unpack --input-dir ./packed --output ./restored
+```
+
+Unpacking validates the complete versioned schema before transactionally
+publishing a restored session, and retains the supported unversioned HDF5 read
+path. It accepts `--progress/--no-progress`; the batch line reports the same
+filled rail as the download command, driven by restored sessions, plus the
+session count, the restored frame count, and the elapsed time:
+
+```console
+Unpacking  ━━━━━━━━━━━━━───────────────────────────   33%  3/9  1200 frames restored  00:42
+packed/episode_000003.hdf5 -> restored/episode_000003
+```
+
+The rail is empty until the first callback reports how many sessions the input
+holds, and shows `?%` until then. That batch has no rate, so it has no estimate
+and only one clock, printed bare because there is nothing to tell it apart from.
+The frame count is the running total over the whole batch, not the count for the
+session named on the line below it, which is why it is spelled out as `frames
+restored`: it only ever counts sessions that finished, so a failed session adds
+nothing, and it reaches its final value in the summary line. It is always printed,
+even at zero, so the line keeps one shape from the first frame to the last.
+Batches continue after an item failure but exit with status 1 if any item fails.
+
+## Convert
+
+`convert` is kept as a compatibility alias for `unpack`, because pipelines that
+were written against the earlier releases still call it. It takes the same
+options and performs the same restore:
+
+```console
+dexverse-dl convert --input ./packed/session.hdf5 --output ./restored
+dexverse-dl convert --input-dir ./packed --output ./restored
+```
+
+Its only differences are the labels on the progress line and in the summary:
+`Converting` and `converted=` instead of `Unpacking` and `unpacked=`. New
+scripts should prefer `unpack`.
+
+## Code layout
+
+`cli.py` only wires typer to the chains; everything else lives in the package for
+the chain it belongs to, and the two chains never import each other.
+
+```
+dexverse_cli/
+  cli.py                 download / unpack / convert, and nothing else
+  common/                what both chains share: the .dexverse state directory, digests
+  progress/              what a running command shows: views, the rail, the reporters
+    view.py              the frozen snapshots a batch hands to its display
+    bar.py               the rail's glyphs, geometry, and styling
+    formatting.py        byte counts and clocks as the lines show them
+    reporter.py          the live two-line region and the reports that drive it
+  download/              manifest -> plan -> one file -> batch -> runner
+  restore/               session format -> raw layout -> publish -> batch -> runner
+```
+
+Each chain follows the same shape, so a reader who knows one knows the other: an
+input contract (`download/manifest.py`, `restore/schema.py`), a plan
+(`download/filesystem.py`, `restore/versioned.py`), the work for one item
+(`download/transfer.py` `FileTransfer`, `restore/session.py` `SessionRestore`), the
+one owner of the batch's state and counters (`*/batch.py`), and the orchestration
+that decides the order (`*/runner.py`).
+

{dexverse_cli-0.1.3.post1 → dexverse_cli-0.1.4}/pyproject.toml

@@ -1,68 +1,69 @@
-[build-system]
-requires = ["hatchling>=1.27"]
-build-backend = "hatchling.build"
-
-[project]
-name = "dexverse-cli"
-version = "0.1.3.post1"
-description = "Download, pack, and unpack DexVerse datasets"
-readme = "src/dexverse_cli/README.md"
-requires-python = ">=3.10"
-dependencies = [
-  "av>=14,<17",
-  "h5py>=3.11,<4",
-  "httpx>=0.28,<1",
-  "numpy>=1.26,<3",
-  "pillow>=10,<13",
-  "rich>=13.9,<15",
-  "typer>=0.16,<1",
-]
-
-[project.optional-dependencies]
-jetson = [
-  "PyGObject>=3.42,<4; platform_system == 'Linux'",
-]
-
-[project.scripts]
-dexverse-dl = "dexverse_cli.cli:app"
-
-[dependency-groups]
-dev = [
-  "mypy>=1.15,<2",
-  "pytest>=8.3,<10",
-  "ruff>=0.11,<1",
-]
-
-[tool.hatch.build.targets.wheel]
-packages = ["src/dexverse_cli"]
-
-[tool.hatch.build.targets.sdist]
-include = [
-  "/pyproject.toml",
-  "/uv.lock",
-  "/.python-version",
-  "/src/dexverse_cli",
-]
-
-[tool.pytest.ini_options]
-addopts = "-ra"
-testpaths = ["tests"]
-
-[tool.ruff]
-target-version = "py310"
-line-length = 120
-
-[tool.ruff.lint]
-select = ["E", "F", "I", "UP", "B", "SIM", "RUF", "TID252"]
-
-[tool.ruff.lint.flake8-tidy-imports]
-ban-relative-imports = "all"
-
-[tool.ruff.lint.per-file-ignores]
-"tests/**/*.py" = ["TID252"]
-
-[tool.mypy]
-python_version = "3.10"
-strict = true
-warn_unreachable = true
-ignore_missing_imports = true
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[project]
+name = "dexverse-cli"
+dynamic = ["version"]
+description = "Download DexVerse dataset manifests and unpack HDF5 sessions"
+readme = "src/dexverse_cli/README.md"
+requires-python = ">=3.10"
+dependencies = [
+  "av>=14,<17",
+  "h5py>=3.11,<4",
+  "httpx>=0.28,<1",
+  "numpy>=1.26,<3",
+  "packaging>=24,<27",
+  "pillow>=10,<13",
+  "rich>=13.9,<15",
+  "typer>=0.16,<1",
+]
+
+[project.scripts]
+dexverse-dl = "dexverse_cli.cli:app"
+
+[dependency-groups]
+dev = [
+  "mypy>=1.15,<2",
+  "pytest>=8.3,<10",
+  "ruff>=0.11,<1",
+]
+
+[tool.hatch.version]
+# One version, declared once: the build backend reads the package marker, so a
+# release bump cannot leave the wheel metadata disagreeing with the code.
+path = "src/dexverse_cli/__init__.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dexverse_cli"]
+
+[tool.hatch.build.targets.sdist]
+include = [
+  "/pyproject.toml",
+  "/uv.lock",
+  "/.python-version",
+  "/src/dexverse_cli",
+]
+
+[tool.pytest.ini_options]
+addopts = "-ra"
+testpaths = ["tests"]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 120
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM", "RUF", "TID252"]
+
+[tool.ruff.lint.flake8-tidy-imports]
+ban-relative-imports = "all"
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**/*.py" = ["TID252"]
+
+[tool.mypy]
+python_version = "3.10"
+strict = true
+warn_unreachable = true
+ignore_missing_imports = true