euler-loading 2.26.0__tar.gz → 2.28.0__tar.gz

{euler_loading-2.26.0 → euler_loading-2.28.0}/.gitignore

@@ -42,3 +42,8 @@ TASK.md
 
 # uv
 uv.lock
+
+# Stray npm lockfile. Nothing in this package is built with npm, and worktree
+# tooling rewrites its "name" field to the branch directory on every branch,
+# so tracking it produced a merge conflict on every quick-merge.
+package-lock.json

{euler_loading-2.26.0 → euler_loading-2.28.0}/CHANGELOG.md

@@ -1,6 +1,21 @@
 # Changelog
 
-## Unreleased
+## 2.27.0 (unreleased)
+
+- Add a dry-run command: `euler-loading <path>` (also `python -m
+  euler_loading <path>`) walks the ds-crawler artifact sets at or below one
+  folder, resolves the loader each dataset head declares in
+  `addons.euler_loading`, decodes a sample of the indexed files with it, and
+  reports shapes, dtypes and value ranges. Directories and `.zip` archives
+  are treated alike, nothing is written, and the exit status is non-zero when
+  a modality fails to resolve or decode. `euler_loading.dry_run.dry_run()`
+  exposes the same report to Python.
+- `resolve_loader_module` and `resolve_writer_module` take a
+  `variant="gpu"|"cpu"` keyword, so the CPU modules can be resolved through
+  the same contract pathway as the torch ones. This is what the dry run's
+  `--cpu` uses to check archives on a torch-free install.
+
+## 2.26.0
 
 - Add `read_extrinsics` to the Synscapes loaders, building a 4x4 rigid
   transform from the six `camera.extrinsic` scalars. `transform_direction`

{euler_loading-2.26.0 → euler_loading-2.28.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: euler-loading
-Version: 2.26.0
+Version: 2.28.0
 Summary: Multi-modal PyTorch dataloader using ds-crawler indices
 Project-URL: Homepage, https://github.com/d-rothen/euler-loading
 Project-URL: Repository, https://github.com/d-rothen/euler-loading
@@ -174,6 +174,36 @@ when you want the CPU variant or a custom callable. See
 [Automatic loader resolution](docs/loaders.md#automatic-loader-resolution) for
 the full contract and writer rules.
 
+## Checking a dataset
+
+Before any of that runs, point the bundled command at a folder to see what
+euler-loading would make of it:
+
+```bash
+euler-loading /data/vkitti2
+```
+
+For every ds-crawler artifact set below that path it resolves the loader the
+dataset declares, decodes a sample of the indexed files with it, and prints
+the shape, dtype and value range that came back:
+
+```text
+vkitti_2.0.3_rgb  [ok]
+  path      /data/vkitti2/vkitti_2.0.3_rgb
+  contract  vkitti2_rgb, "Virtual KITTI 2 RGB", modality key: rgb
+  loader    vkitti2.rgb -> euler_loading.loaders.gpu.vkitti2.rgb
+  index     21260 files; available splits: train, val
+  decoded   1/1
+    Scene01/clone/frames/rgb/Camera_0/rgb_00000.jpg  ->  Tensor (3, 375, 1242) float32 in [0, 1]  (11.4 ms)
+```
+
+Nothing is written, directories and `.zip` archives are treated alike, and the
+exit status is non-zero if any modality failed to resolve or decode — so it
+also works as a CI step. `--all` decodes every file instead of a sample,
+`--cpu` checks the NumPy loaders on a torch-free install, and `--json` prints
+the same report for a machine. See
+[Dry-running loaders](docs/loaders.md#dry-running-loaders).
+
 ## What you get
 
 | | |
@@ -186,6 +216,7 @@ the full contract and writer rules.
 | **Loader resolution** | Loaders and writers resolve from the `dataset-head.json` `addons.euler_loading` contract, so datasets describe how to read themselves. |
 | **Writing back** | Resolved writers put inference outputs back in dataset-native formats, re-indexable with matching IDs. |
 | **Spatial preprocessing** | `SamplePreprocessor` resizes and crops consistently across images, depth, masks, ray maps *and* intrinsics. |
+| **Dry runs** | `euler-loading <folder>` resolves and exercises every loader a dataset declares, without writing anything. |
 
 ## Built-in loaders
 

{euler_loading-2.26.0 → euler_loading-2.28.0}/README.md

@@ -135,6 +135,36 @@ when you want the CPU variant or a custom callable. See
 [Automatic loader resolution](docs/loaders.md#automatic-loader-resolution) for
 the full contract and writer rules.
 
+## Checking a dataset
+
+Before any of that runs, point the bundled command at a folder to see what
+euler-loading would make of it:
+
+```bash
+euler-loading /data/vkitti2
+```
+
+For every ds-crawler artifact set below that path it resolves the loader the
+dataset declares, decodes a sample of the indexed files with it, and prints
+the shape, dtype and value range that came back:
+
+```text
+vkitti_2.0.3_rgb  [ok]
+  path      /data/vkitti2/vkitti_2.0.3_rgb
+  contract  vkitti2_rgb, "Virtual KITTI 2 RGB", modality key: rgb
+  loader    vkitti2.rgb -> euler_loading.loaders.gpu.vkitti2.rgb
+  index     21260 files; available splits: train, val
+  decoded   1/1
+    Scene01/clone/frames/rgb/Camera_0/rgb_00000.jpg  ->  Tensor (3, 375, 1242) float32 in [0, 1]  (11.4 ms)
+```
+
+Nothing is written, directories and `.zip` archives are treated alike, and the
+exit status is non-zero if any modality failed to resolve or decode — so it
+also works as a CI step. `--all` decodes every file instead of a sample,
+`--cpu` checks the NumPy loaders on a torch-free install, and `--json` prints
+the same report for a machine. See
+[Dry-running loaders](docs/loaders.md#dry-running-loaders).
+
 ## What you get
 
 | | |
@@ -147,6 +177,7 @@ the full contract and writer rules.
 | **Loader resolution** | Loaders and writers resolve from the `dataset-head.json` `addons.euler_loading` contract, so datasets describe how to read themselves. |
 | **Writing back** | Resolved writers put inference outputs back in dataset-native formats, re-indexable with matching IDs. |
 | **Spatial preprocessing** | `SamplePreprocessor` resizes and crops consistently across images, depth, masks, ray maps *and* intrinsics. |
+| **Dry runs** | `euler-loading <folder>` resolves and exercises every loader a dataset declares, without writing anything. |
 
 ## Built-in loaders
 

{euler_loading-2.26.0 → euler_loading-2.28.0}/docs/README.md

@@ -3,7 +3,7 @@
 | Guide | Covers |
 |---|---|
 | [Dataset & modalities](dataset.md) | `Modality` and `MultiModalDataset` reference, the sample dict, hierarchical modalities, splits, scoped metadata, zip archives, layout-aware loading |
-| [Loaders & writers](loaders.md) | The loader contract, per-file attributes, automatic resolution, protocols, and the full built-in loader inventory |
+| [Loaders & writers](loaders.md) | The loader contract, per-file attributes, automatic resolution, the `euler-loading` dry run, protocols, and the full built-in loader inventory |
 | [Preprocessing & transforms](preprocessing.md) | Cross-modal transforms, `SamplePreprocessor`, field kinds, calibration-aware resize and crop |
 | [Writing outputs](writing.md) | Writing predictions back in dataset-native formats and re-indexing them |
 

{euler_loading-2.26.0 → euler_loading-2.28.0}/docs/loaders.md

@@ -7,6 +7,7 @@ that file means.
 - [The contract](#the-contract)
 - [Per-file attributes](#per-file-attributes)
 - [Automatic loader resolution](#automatic-loader-resolution)
+- [Dry-running loaders](#dry-running-loaders)
 - [Loader protocols](#loader-protocols)
 - [Built-in loaders](#built-in-loaders)
 
@@ -129,6 +130,102 @@ Writers resolve from the same `addons.euler_loading` entry, in this order:
 2. for `function: "read_<suffix>"`, `write_<suffix>`,
 3. `write_<function>`.
 
+## Dry-running loaders
+
+Resolution only pays off if the loader a dataset names can really read its
+files. Point the bundled command at a path to find out, before a training run
+depends on it:
+
+```bash
+euler-loading /data/vkitti2
+python -m euler_loading /data/vkitti2   # same thing, no console script needed
+```
+
+It takes one path — a modality root, a `.zip` archive, or a folder holding
+several of them — and for every ds-crawler artifact set below it:
+
+1. loads the index, or a named split,
+2. resolves the loader its `addons.euler_loading` entry declares,
+3. decodes a sample of the indexed files with that loader,
+4. reports the shape, dtype and value range of what came back.
+
+Nothing is written and no dataset is constructed. A modality that cannot be
+read is reported, not raised, so one broken archive does not hide the rest.
+
+```text
+euler-loading dry-run: /data/vkitti2
+loader variant: gpu (torch tensors)
+
+vkitti_2.0.3_rgb  [ok]
+  path      /data/vkitti2/vkitti_2.0.3_rgb
+  contract  vkitti2_rgb, "Virtual KITTI 2 RGB", modality key: rgb
+  loader    vkitti2.rgb -> euler_loading.loaders.gpu.vkitti2.rgb
+  writer    write_rgb
+  index     21260 files; available splits: train, val
+  decoded   1/1
+    Scene01/clone/frames/rgb/Camera_0/rgb_00000.jpg  ->  Tensor (3, 375, 1242) float32 in [0, 1]  (11.4 ms)
+
+vkitti_2.0.3_textgt  [ok]
+  path      /data/vkitti2/vkitti_2.0.3_textgt
+  contract  vkitti2_intrinsics, "Virtual KITTI 2 intrinsics", modality key: camera_intrinsics
+  loader    vkitti2.read_intrinsics -> euler_loading.loaders.gpu.vkitti2.read_intrinsics (hierarchical)
+  writer    write_intrinsics
+  index     50 files
+  decoded   1/1
+    Scene01/clone/intrinsic.txt  ->  Tensor (3, 3) float32 in [0, 725]  (0.6 ms)
+
+2 modalities checked: 2 ok, 0 failed; 2/2 files decoded
+```
+
+The exit status is 0 when every modality resolved a loader and decoded every
+file it was asked to, and 1 otherwise, so the command works as a CI step or a
+job prologue.
+
+| Flag | Effect |
+|---|---|
+| `-n, --samples N` | Files to decode per modality, spread evenly across the index (default 1). `0` resolves loaders without decoding anything. |
+| `--all` | Decode every indexed file. Slow, but checks the whole archive. |
+| `--split NAME` | Dry-run a named ds-crawler split instead of the canonical index. |
+| `--scope SCOPE` | Read `.ds_crawler/SCOPE/` only, instead of every artifact set a root carries. |
+| `--cpu` | Resolve the NumPy loaders instead of the torch ones. Works on an install without PyTorch. |
+| `--max-depth N` | Directory levels below the path to search for dataset roots (default 3). |
+| `--json` | Print the report as JSON instead of text. |
+| `-v, --verbose` | Log debug output, including a traceback for every failed load. |
+
+The path takes the same inline selectors as `Modality`, so
+`euler-loading /data/muses.zip:train#scope=rgb` checks one split of one scope.
+
+Discovery stops at the first dataset root down each branch: a directory
+carrying its own `.ds_crawler/` is checked, never descended into, so a
+dataset's own scene directories are never walked. A root with several
+metadata scopes contributes one entry per scope, labelled with the selector
+that addresses it.
+
+### Reading the report
+
+| Line | What it is telling you |
+|---|---|
+| `error loader: ... no 'addons.euler_loading' entry` | the dataset head declares no loader, so only an explicit `Modality(..., loader=...)` can read it |
+| `loader vkitti2.rgb (unresolved)` plus an `error` | the contract names a loader that is unknown, misspelled, or needs a dependency this install lacks — `--cpu` resolves the torch-free variant |
+| `decoded 0/2` with per-file `FAILED` | the files are indexed and present, but the declared loader cannot decode them |
+| `(hierarchical)` after the loader | the declared function reads per-scene data; pass that modality as `hierarchical_modalities`, not `modalities` |
+| `error index: ...` | ds-crawler could not produce an index — a missing split, an unreadable archive, or a dataset head the contract rejects |
+| a warning about shared file IDs | the modalities below the path have no file ID in common, so they cannot be intersected into one `MultiModalDataset` |
+
+The same report is available as an API, which is what the command is a thin
+shell around:
+
+```python
+from euler_loading.dry_run import dry_run
+
+report = dry_run("/data/vkitti2", samples=3)
+if not report.ok:
+    for modality in report.modalities:
+        print(modality.label, modality.errors)
+
+report.to_dict()   # the same structure --json prints
+```
+
 ## Loader protocols
 
 `DenseDepthLoader` is a `runtime_checkable` Protocol defining the loader

euler_loading-2.28.0/euler_loading/__main__.py

@@ -0,0 +1,10 @@
+"""``python -m euler_loading`` — dry-run the loaders a dataset declares."""
+
+from __future__ import annotations
+
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

{euler_loading-2.26.0 → euler_loading-2.28.0}/euler_loading/_ds_crawler_utils.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import io
+import zipfile
 from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
@@ -22,6 +24,22 @@ from ds_crawler.zip_utils import (
 )
 
 
+def read_zip_member(
+    archive: zipfile.ZipFile,
+    prefix: str,
+    relative_path: str,
+) -> io.BytesIO:
+    """Read one member of a zip-backed modality into an in-memory buffer.
+
+    The buffer carries the modality-relative path as its ``name`` so loaders
+    that branch on the file extension behave the same as for a filesystem
+    path.
+    """
+    buffer = io.BytesIO(archive.read(prefix + relative_path))
+    buffer.name = relative_path
+    return buffer
+
+
 def as_non_empty_str(value: Any) -> str | None:
     if value is None:
         return None

{euler_loading-2.26.0 → euler_loading-2.28.0}/euler_loading/_resolution.py

@@ -18,6 +18,10 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+#: Built-in loader variants. ``gpu`` returns torch tensors, ``cpu`` NumPy
+#: arrays; automatic resolution from a dataset contract uses ``gpu``.
+LOADER_VARIANTS = ("gpu", "cpu")
+
 _LOADER_MODULES: dict[str, str] = {
     "materialized": "euler_loading.loaders.materialized",
     "vkitti2": "euler_loading.loaders.gpu.vkitti2",
@@ -46,30 +50,45 @@ def _get_euler_loading_meta(index: Mapping[str, Any]) -> Mapping[str, Any] | Non
     return None
 
 
-def resolve_loader_module(name: str) -> ModuleType:
-    """Import and return the GPU loader module for *name*.
+def resolve_loader_module(name: str, *, variant: str = "gpu") -> ModuleType:
+    """Import and return the loader module for *name*.
 
     Example::
 
         module = resolve_loader_module("vkitti2")
         sky_fn = module.sky_mask  # get a specific function
 
+    Args:
+        name: Loader name, as declared by ``addons.euler_loading.loader``.
+        variant: ``"gpu"`` for the torch loaders (the default, and what
+            automatic resolution uses) or ``"cpu"`` for the NumPy ones.
+
     Raises:
-        ValueError: If *name* does not match any known loader.
+        ValueError: If *name* does not match any known loader, or *variant*
+            is neither ``"gpu"`` nor ``"cpu"``.
     """
+    if variant not in LOADER_VARIANTS:
+        available = ", ".join(LOADER_VARIANTS)
+        raise ValueError(
+            f"Unknown loader variant {variant!r}. Available variants: {available}"
+        )
     module_path = _LOADER_MODULES.get(name)
     if module_path is None:
         available = ", ".join(sorted(_LOADER_MODULES))
         raise ValueError(f"Unknown loader {name!r}. Available loaders: {available}")
+    if variant == "cpu":
+        # Loaders that exist in one variant only (``materialized``) are not
+        # registered under ``.gpu.`` and therefore stay as they are.
+        module_path = module_path.replace(".loaders.gpu.", ".loaders.cpu.")
     return importlib.import_module(module_path)
 
 
-def resolve_writer_module(name: str) -> ModuleType:
+def resolve_writer_module(name: str, *, variant: str = "gpu") -> ModuleType:
     """Import and return the writer module for *name*.
 
     Writers live next to loader functions in the same modules.
     """
-    return resolve_loader_module(name)
+    return resolve_loader_module(name, variant=variant)
 
 
 def _builtin_loader_id(loader: Callable[..., Any]) -> str | None:
@@ -119,8 +138,13 @@ def _resolve_loader(
     modality_name: str,
     modality: Modality,
     index: dict[str, Any],
+    variant: str = "gpu",
 ) -> Callable[..., Any]:
-    """Return the effective loader for a modality."""
+    """Return the effective loader for a modality.
+
+    An explicit ``Modality.loader`` always wins; *variant* only selects
+    between the built-in GPU and CPU modules during automatic resolution.
+    """
     if modality.loader is not None:
         return modality.loader
 
@@ -143,7 +167,7 @@ def _resolve_loader(
     module_name: str = euler_loading_meta["loader"]
     func_name: str = euler_loading_meta["function"]
 
-    module = resolve_loader_module(module_name)
+    module = resolve_loader_module(module_name, variant=variant)
 
     func = getattr(module, func_name, None)
     if func is None or not callable(func):
@@ -185,6 +209,7 @@ def _resolve_writer(
     modality_name: str,
     modality: Modality,
     index: dict[str, Any],
+    variant: str = "gpu",
 ) -> Callable[..., Any] | None:
     """Return the effective writer for a modality, if available."""
     if modality.writer is not None:
@@ -200,7 +225,7 @@ def _resolve_writer(
         return None
 
     try:
-        module = resolve_writer_module(module_name)
+        module = resolve_writer_module(module_name, variant=variant)
     except ValueError:
         logger.warning(
             "Modality '%s': cannot resolve writer module %r.",