euler-loading 2.1.0__tar.gz → 2.2.1__tar.gz

{euler_loading-2.1.0 → euler_loading-2.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: euler-loading
-Version: 2.1.0
+Version: 2.2.1
 Summary: Multi-modal PyTorch dataloader using ds-crawler indices
 Requires-Python: >=3.9
 Requires-Dist: ds-crawler

{euler_loading-2.1.0 → euler_loading-2.2.1}/README.md

@@ -218,6 +218,44 @@ def mask_sky_in_depth(sample: dict) -> dict:
     return sample
 ```
 
+### Built-in spatial preprocessing
+
+`euler_loading.SamplePreprocessor` applies shared spatial ops such as resize and crop
+consistently across sample fields, including calibration-sensitive fields such as
+camera intrinsics and ray maps.
+
+```python
+from euler_loading import FieldSpec, SamplePreprocessor
+
+preprocessor = SamplePreprocessor.from_config(
+    {
+        "resize": [384, 768],
+        "crop": {"size": [320, 640], "anchor": "center"},
+        "fields": {
+            "rgb": {"kind": "image"},
+            "depth": {"kind": "depth"},
+            "valid_mask": {"kind": "mask"},
+            "intrinsics": {"kind": "intrinsics", "reduce": "first"},
+            "ray_map": {"kind": "ray_map"},
+        },
+    }
+)
+
+dataset = MultiModalDataset(
+    modalities={...},
+    hierarchical_modalities={...},
+    transforms=[preprocessor],
+)
+```
+
+- `kind="image"` and `kind="depth"` default to bilinear interpolation.
+- `kind="mask"` defaults to nearest-neighbour interpolation and preserves mask dtype.
+- `kind="ray_map"` bilinearly resizes and renormalizes vectors to unit length.
+- `kind="intrinsics"` rescales the camera matrix for resize and shifts the principal point for crop.
+- `reduce="first"` is useful for hierarchical modalities such as one intrinsics file per scene/camera.
+
+If you already set `modality_type` on your `Modality(...)` definitions, the preprocessor can bind to the dataset and reuse those hints automatically.
+
 ## Zip archive support
 
 Modality paths can point to `.zip` files instead of directories. euler-loading detects zip paths automatically and reads files directly from the archive without extraction:

{euler_loading-2.1.0 → euler_loading-2.2.1}/euler_loading/__init__.py

@@ -6,15 +6,31 @@ from ._writing import create_dataset_writer_from_index
 from .dataset import Modality, MultiModalDataset
 from .indexing import FileRecord
 from .loaders.contracts import DenseDepthCodec, DenseDepthLoader, DenseDepthWriter
+from .preprocessing import (
+    Crop,
+    FieldSpec,
+    Resize,
+    SamplePreprocessor,
+    crop_intrinsics,
+    infer_field_spec,
+    resize_intrinsics,
+)
 
 __all__ = [
     "DenseDepthCodec",
     "DenseDepthLoader",
     "DenseDepthWriter",
     "FileRecord",
+    "Crop",
+    "FieldSpec",
     "Modality",
     "MultiModalDataset",
+    "Resize",
+    "SamplePreprocessor",
     "create_dataset_writer_from_index",
+    "crop_intrinsics",
+    "infer_field_spec",
     "resolve_loader_module",
     "resolve_writer_module",
+    "resize_intrinsics",
 ]

{euler_loading-2.1.0 → euler_loading-2.2.1}/euler_loading/_ds_crawler_utils.py

@@ -49,7 +49,13 @@ def first_non_empty_list(*candidates: list[str] | None) -> list[str]:
 
 
 def extract_ds_crawler_properties(index_output: Mapping[str, Any]) -> dict[str, Any]:
-    return get_dataset_contract(dict(index_output)).to_properties_dict()
+    try:
+        return get_dataset_contract(dict(index_output)).to_properties_dict()
+    except Exception:
+        properties = index_output.get("properties")
+        if isinstance(properties, Mapping):
+            return dict(properties)
+        return {}
 
 
 _SPLIT_NAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
@@ -126,13 +132,18 @@ def load_index_output(
 
     normalized_split = validate_split_name(split)
     if _load_dataset_split is not None:
-        return _load_dataset_split(
-            path,
-            normalized_split,
-            strict=strict,
-            save_index=save_index,
-            force_reindex=force_reindex,
-        )
+        try:
+            return _load_dataset_split(
+                path,
+                normalized_split,
+                strict=strict,
+                save_index=save_index,
+                force_reindex=force_reindex,
+            )
+        except FileNotFoundError:
+            # Fallback for mocked tests or callers that provide synthetic index
+            # loaders without a full ds-crawler config on disk.
+            pass
 
     base_output = index_dataset_from_path_fn(
         path,
@@ -149,5 +160,5 @@ def load_index_output(
 
     result = dict(base_output)
     result["index"] = split_dataset
-    result.pop("dataset", None)
+    result["dataset"] = split_dataset
     return result

{euler_loading-2.1.0 → euler_loading-2.2.1}/euler_loading/_resolution.py

@@ -25,6 +25,21 @@ _LOADER_MODULES: dict[str, str] = {
 }
 
 
+def _get_euler_loading_meta(index: Mapping[str, Any]) -> Mapping[str, Any] | None:
+    try:
+        contract = get_dataset_contract(dict(index))
+        euler_loading_meta = contract.get_addon("euler_loading")
+    except Exception:
+        euler_loading_meta = None
+    if isinstance(euler_loading_meta, Mapping):
+        return euler_loading_meta
+
+    raw_meta = index.get("euler_loading")
+    if isinstance(raw_meta, Mapping):
+        return raw_meta
+    return None
+
+
 def resolve_loader_module(name: str) -> ModuleType:
     """Import and return the GPU loader module for *name*.
 
@@ -63,11 +78,7 @@ def _resolve_loader(
     if modality.loader is not None:
         return modality.loader
 
-    try:
-        contract = get_dataset_contract(index)
-        euler_loading_meta = contract.get_addon("euler_loading")
-    except Exception:
-        euler_loading_meta = None
+    euler_loading_meta = _get_euler_loading_meta(index)
     if not isinstance(euler_loading_meta, Mapping):
         raise ValueError(
             f"Modality {modality_name!r}: no explicit loader provided and the "
@@ -132,11 +143,7 @@ def _resolve_writer(
     if modality.writer is not None:
         return modality.writer
 
-    try:
-        contract = get_dataset_contract(index)
-        euler_loading_meta = contract.get_addon("euler_loading")
-    except Exception:
-        euler_loading_meta = None
+    euler_loading_meta = _get_euler_loading_meta(index)
     if not isinstance(euler_loading_meta, Mapping):
         return None
 

{euler_loading-2.1.0 → euler_loading-2.2.1}/euler_loading/_writing.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
+import json
 import logging
 import os
 import tempfile
+import zipfile
 from collections.abc import Mapping
 from pathlib import Path
 from typing import Any, Callable
@@ -18,10 +20,93 @@ from .loaders._writer_utils import supports_stream_target
 
 logger = logging.getLogger(__name__)
 
+_DEFAULT_WRITER_INDEX_FILENAME = "output.json"
+
 
 OutputDestination = str | os.PathLike[str] | DatasetWriter | ZipDatasetWriter
 
 
+def _wrap_writer_save_index(
+    writer: DatasetWriter | ZipDatasetWriter,
+) -> DatasetWriter | ZipDatasetWriter:
+    original_save_index = writer.save_index
+
+    def _build_legacy_fields() -> dict[str, Any]:
+        output = writer.build_output()
+        result: dict[str, Any] = {}
+
+        head = output.get("head")
+        if isinstance(head, Mapping):
+            dataset = head.get("dataset")
+            modality = head.get("modality")
+            addons = head.get("addons")
+
+            if isinstance(dataset, Mapping):
+                name = as_non_empty_str(dataset.get("name"))
+                if name is not None:
+                    result["name"] = name
+
+            if isinstance(modality, Mapping):
+                modality_key = as_non_empty_str(modality.get("key"))
+                if modality_key is not None:
+                    result["type"] = modality_key
+                meta = modality.get("meta")
+                if isinstance(meta, Mapping):
+                    result["meta"] = dict(meta)
+
+            if isinstance(addons, Mapping):
+                for addon_name in ("euler_train", "euler_loading"):
+                    addon = addons.get(addon_name)
+                    if isinstance(addon, Mapping):
+                        result[addon_name] = dict(addon)
+
+        index_tree = output.get("index")
+        if isinstance(index_tree, Mapping):
+            result["dataset"] = dict(index_tree)
+
+        return result
+
+    def save_index(filename: str = _DEFAULT_WRITER_INDEX_FILENAME):
+        legacy_fields = _build_legacy_fields()
+        path = original_save_index(filename=filename)
+
+        if isinstance(writer, ZipDatasetWriter):
+            entry_name = f".ds_crawler/{filename}"
+            archive_path = Path(writer.root)
+            with zipfile.ZipFile(archive_path, "r") as source_zip:
+                output_index = json.loads(source_zip.read(entry_name).decode("utf-8"))
+                output_index.update(legacy_fields)
+
+                replacement_name = archive_path.with_suffix(archive_path.suffix + ".tmp")
+                with zipfile.ZipFile(replacement_name, "w") as target_zip:
+                    replaced = False
+                    for info in source_zip.infolist():
+                        data = source_zip.read(info.filename)
+                        if info.filename == entry_name and not replaced:
+                            data = json.dumps(output_index, indent=2).encode("utf-8")
+                            replaced = True
+                        target_zip.writestr(info, data)
+
+                    if not replaced:
+                        target_zip.writestr(
+                            entry_name,
+                            json.dumps(output_index, indent=2),
+                            compress_type=zipfile.ZIP_DEFLATED,
+                        )
+
+            replacement_name.replace(archive_path)
+            return path
+
+        output_path = Path(path)
+        output_index = json.loads(output_path.read_text(encoding="utf-8"))
+        output_index.update(legacy_fields)
+        output_path.write_text(json.dumps(output_index, indent=2), encoding="utf-8")
+        return path
+
+    setattr(writer, "save_index", save_index)
+    return writer
+
+
 def create_dataset_writer_from_index(
     *,
     index_output: Mapping[str, Any],
@@ -29,8 +114,6 @@ def create_dataset_writer_from_index(
     zip: bool = False,
 ) -> DatasetWriter | ZipDatasetWriter:
     """Create a ds-crawler writer that mirrors an existing index's metadata."""
-    contract = get_dataset_contract(dict(index_output))
-
     indexing = index_output.get("indexing")
     hierarchy = indexing.get("hierarchy") if isinstance(indexing, Mapping) else None
     id_cfg = indexing.get("id") if isinstance(indexing, Mapping) else None
@@ -39,11 +122,46 @@ def create_dataset_writer_from_index(
         as_non_empty_str(id_cfg.get("join_char")) if isinstance(id_cfg, Mapping) else None,
     )
     writer_cls = ZipDatasetWriter if zip else DatasetWriter
-    return writer_cls(
+    try:
+        contract = get_dataset_contract(dict(index_output))
+    except Exception:
+        contract = None
+
+    if contract is not None:
+        writer = writer_cls(
+            root,
+            head=contract.to_mapping(),
+            separator=separator,
+        )
+        return _wrap_writer_save_index(writer)
+
+    name = as_non_empty_str(index_output.get("name"))
+    type_name = as_non_empty_str(index_output.get("type"))
+    if name is None or type_name is None:
+        raise ValueError(
+            "Cannot create a dataset writer without ds-crawler head metadata. "
+            "Expected either a dataset contract or top-level 'name' and 'type'."
+        )
+
+    euler_train = index_output.get("euler_train")
+    euler_loading = index_output.get("euler_loading")
+    meta = index_output.get("meta")
+
+    properties: dict[str, Any] = {}
+    if isinstance(euler_loading, Mapping):
+        properties["euler_loading"] = dict(euler_loading)
+    if isinstance(meta, Mapping):
+        properties["meta"] = dict(meta)
+
+    writer = writer_cls(
         root,
-        head=contract.to_mapping(),
+        name=name,
+        type=type_name,
+        euler_train=dict(euler_train) if isinstance(euler_train, Mapping) else None,
         separator=separator,
+        **properties,
     )
+    return _wrap_writer_save_index(writer)
 
 
 def _resolve_output_destination(

{euler_loading-2.1.0 → euler_loading-2.2.1}/euler_loading/dataset.py

@@ -59,6 +59,65 @@ __all__ = [
 ]
 
 
+def _callable_name(value: Any) -> str:
+    module = getattr(value, "__module__", None)
+    qualname = getattr(value, "__qualname__", None)
+    if module and qualname:
+        return f"{module}.{qualname}"
+    name = getattr(value, "__name__", None)
+    if module and name:
+        return f"{module}.{name}"
+    cls = value.__class__
+    return f"{cls.__module__}.{cls.__qualname__}"
+
+
+def _describe_transform(transform: Any) -> str:
+    describe = getattr(transform, "describe", None)
+    if callable(describe):
+        try:
+            return str(describe())
+        except Exception:
+            logger.debug(
+                "Failed to describe transform %s; falling back to callable name.",
+                _callable_name(transform),
+                exc_info=True,
+            )
+    return _callable_name(transform)
+
+
+def _get_index_tree(index_output: Mapping[str, Any]) -> dict[str, Any]:
+    """Return the dataset tree from a ds-crawler output payload."""
+    index_tree = index_output.get("index")
+    if isinstance(index_tree, Mapping):
+        dataset_tree = index_tree.get("dataset")
+        if isinstance(dataset_tree, Mapping):
+            return dict(dataset_tree)
+        return dict(index_tree)
+
+    dataset_tree = index_output.get("dataset")
+    if isinstance(dataset_tree, Mapping):
+        return dict(dataset_tree)
+
+    raise KeyError(
+        "ds-crawler output must contain either an 'index' tree or a 'dataset' tree."
+    )
+
+
+def _get_index_meta(index_output: Mapping[str, Any]) -> dict[str, Any] | None:
+    """Return modality meta from a ds-crawler output payload when available."""
+    try:
+        meta = get_dataset_contract(dict(index_output)).meta
+    except Exception:
+        meta = None
+    if isinstance(meta, Mapping):
+        return dict(meta)
+
+    raw_meta = index_output.get("meta")
+    if isinstance(raw_meta, Mapping):
+        return dict(raw_meta)
+    return None
+
+
 #TODO: might want to add slots=True in a 3.10+ only codebase
 @dataclass(frozen=True)
 class Modality:
@@ -320,7 +379,7 @@ class MultiModalDataset(_BaseDataset):
             self._resolved_writers[name] = _resolve_writer(
                 modality_name=name, modality=modality, index=index,
             )
-            records = collect_files(index["index"])
+            records = collect_files(_get_index_tree(index))
             lookup = {
                 "/".join(rec.hierarchy_path + (rec.file_entry["id"],)): rec
                 for rec in records
@@ -352,7 +411,7 @@ class MultiModalDataset(_BaseDataset):
             self._resolved_writers[name] = _resolve_writer(
                 modality_name=name, modality=modality, index=index,
             )
-            files_by_level = collect_hierarchical_files(index["index"])
+            files_by_level = collect_hierarchical_files(_get_index_tree(index))
             self._hierarchical_lookups[name] = files_by_level
             total_files = sum(len(f) for f in files_by_level.values())
             logger.info(
@@ -406,13 +465,28 @@ class MultiModalDataset(_BaseDataset):
         # -- Caches ----------------------------------------------------------
         self._hierarchical_cache: dict[str, Any] = {}
 
+        # -- Optional transform binding -------------------------------------
+        for transform in self._transforms:
+            bind = getattr(transform, "bind_to_dataset", None)
+            if callable(bind):
+                bind(self)
+        if self._transforms:
+            logger.info("Configured %d sample transform(s):", len(self._transforms))
+            for idx, transform in enumerate(self._transforms, start=1):
+                logger.info(
+                    "Transform %d/%d: %s",
+                    idx,
+                    len(self._transforms),
+                    _describe_transform(transform),
+                )
+        else:
+            logger.info("Configured 0 sample transforms.")
+
     def get_modality_metadata(self, modality_name: str) -> dict[str, Any]:
         """Return the metadata dict for a given modality name."""
         index = self._index_outputs.get(modality_name, {})
-        try:
-            return get_dataset_contract(index).meta or {}
-        except Exception:
-            return {}
+        meta = _get_index_meta(index)
+        return meta or {}
 
     def get_modality_index(self, modality_name: str) -> dict[str, Any]:
         """Return the cached ds-crawler index for a modality."""
@@ -510,12 +584,7 @@ class MultiModalDataset(_BaseDataset):
                 output_root=output_root, modality_name=modality_name
             )
             record = self._lookups[modality_name][sample_id]
-            try:
-                modality_meta = get_dataset_contract(
-                    self._index_outputs[modality_name]
-                ).meta
-            except Exception:
-                modality_meta = None
+            modality_meta = _get_index_meta(self._index_outputs[modality_name])
             basename = Path(record.file_entry["path"]).name
             relative_path = record.file_entry["path"]
             full_id = _build_writer_full_id(
@@ -558,7 +627,8 @@ class MultiModalDataset(_BaseDataset):
             try:
                 name = get_dataset_contract(index).name
             except Exception:
-                name = None
+                raw_name = index.get("name")
+                name = str(raw_name) if raw_name is not None else None
             if name is None:
                 continue
             if first_name is None:
@@ -617,12 +687,7 @@ class MultiModalDataset(_BaseDataset):
 
         for name, modality in self._modalities.items():
             record = self._lookups[name][sample_id]
-            try:
-                modality_meta = get_dataset_contract(
-                    self._index_outputs[name]
-                ).meta
-            except Exception:
-                modality_meta = None
+            modality_meta = _get_index_meta(self._index_outputs[name])
 
             if name in self._zip_modalities:
                 file_or_path = self._open_from_zip(
@@ -643,12 +708,7 @@ class MultiModalDataset(_BaseDataset):
         for name, modality in self._hierarchical_modalities.items():
             files_by_level = self._hierarchical_lookups[name]
             matched = match_hierarchical_files(hierarchy_path, files_by_level)
-            try:
-                modality_meta = get_dataset_contract(
-                    self._hierarchical_index_outputs[name]
-                ).meta
-            except Exception:
-                modality_meta = None
+            modality_meta = _get_index_meta(self._hierarchical_index_outputs[name])
             loaded: dict[str, Any] = {}
             for entry in matched:
                 cache_key = f"{modality.path}/{entry['path']}"