euler-loading 2.2.2__tar.gz → 2.4.0__tar.gz

{euler_loading-2.2.2 → euler_loading-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: euler-loading
-Version: 2.2.2
+Version: 2.4.0
 Summary: Multi-modal PyTorch dataloader using ds-crawler indices
 Requires-Python: >=3.9
 Requires-Dist: ds-crawler

{euler_loading-2.2.2 → euler_loading-2.4.0}/README.md

@@ -218,6 +218,19 @@ def mask_sky_in_depth(sample: dict) -> dict:
     return sample
 ```
 
+For common mask-driven replacements, `MaskedValueOverride` can do the same
+without writing a custom callable:
+
+```python
+from euler_loading import MaskedValueOverride
+
+replace_sky_depth = MaskedValueOverride(
+    target_key="depth",
+    mask_key="sky_mask",
+    value=300.0,
+)
+```
+
 ### Built-in spatial preprocessing
 
 `euler_loading.SamplePreprocessor` applies shared spatial ops such as resize and crop

{euler_loading-2.2.2 → euler_loading-2.4.0}/euler_loading/__init__.py

@@ -9,6 +9,7 @@ from .loaders.contracts import DenseDepthCodec, DenseDepthLoader, DenseDepthWrit
 from .preprocessing import (
     Crop,
     FieldSpec,
+    MaskedValueOverride,
     Resize,
     SamplePreprocessor,
     crop_intrinsics,
@@ -23,6 +24,7 @@ __all__ = [
     "FileRecord",
     "Crop",
     "FieldSpec",
+    "MaskedValueOverride",
     "Modality",
     "MultiModalDataset",
     "Resize",

{euler_loading-2.2.2 → euler_loading-2.4.0}/euler_loading/dataset.py

@@ -118,6 +118,36 @@ def _get_index_meta(index_output: Mapping[str, Any]) -> dict[str, Any] | None:
     return None
 
 
+def _extract_id_schema(index_output: Mapping[str, Any]) -> dict[str, Any]:
+    """Pull id/hierarchy separators from a ds-crawler output payload.
+
+    Supports both modern (``indexing.id.join_char``) and legacy
+    (top-level ``id_regex_join_char``) layouts. Missing keys fall back to
+    ds-crawler defaults so the returned dict is always usable.
+    """
+    indexing = index_output.get("indexing")
+    indexing_map = indexing if isinstance(indexing, Mapping) else {}
+    id_cfg = indexing_map.get("id")
+    id_map = id_cfg if isinstance(id_cfg, Mapping) else {}
+    hierarchy_cfg = indexing_map.get("hierarchy")
+    hierarchy_map = hierarchy_cfg if isinstance(hierarchy_cfg, Mapping) else {}
+
+    id_join_char = id_map.get("join_char") or index_output.get("id_regex_join_char") or "+"
+    hierarchy_separator = hierarchy_map.get("separator") or "-"
+    id_regex = id_map.get("regex") or index_output.get("id_regex")
+    hierarchy_regex = hierarchy_map.get("regex") or index_output.get("hierarchy_regex")
+
+    schema: dict[str, Any] = {
+        "hierarchy_separator": hierarchy_separator,
+        "id_join_char": id_join_char,
+    }
+    if id_regex:
+        schema["id_regex"] = id_regex
+    if hierarchy_regex:
+        schema["hierarchy_regex"] = hierarchy_regex
+    return schema
+
+
 #TODO: might want to add slots=True in a 3.10+ only codebase
 @dataclass(frozen=True)
 class Modality:
@@ -488,6 +518,58 @@ class MultiModalDataset(_BaseDataset):
         meta = _get_index_meta(index)
         return meta or {}
 
+    def describe_id_schema(self) -> dict[str, Any]:
+        """Return the id-construction schema used by this dataset.
+
+        Downstream consumers that want to split or match ``sample["full_id"]``
+        / ``sample["id"]`` values back to ds-crawler fields need to know the
+        separators used to build them. This method exposes that schema so it
+        can be embedded in experiment manifests.
+
+        Schema keys:
+            - ``hierarchy_separator``: the character between the named
+              capture group *name* and its *value* within a single id part
+              (ds-crawler ``indexing.hierarchy.separator``, typically ``"-"``).
+            - ``id_join_char``: the character between id parts (ds-crawler
+              ``indexing.id.join_char`` or legacy ``id_regex_join_char``,
+              default ``"+"``).
+            - ``full_id_separator``: the character between hierarchy levels
+              and the file id in ``sample["full_id"]`` (currently ``"/"``).
+            - ``id_regex`` / ``hierarchy_regex``: the raw regexes, when
+              recorded in the ds-crawler output.
+            - ``modalities``: per-modality overrides populated only when
+              modalities disagree on the above.
+
+        Example::
+
+            {
+              "hierarchy_separator": "-",
+              "id_join_char": "+",
+              "full_id_separator": "/",
+              "id_regex": "...",
+            }
+        """
+        per_modality: dict[str, dict[str, Any]] = {}
+        for name, index in self._index_outputs.items():
+            per_modality[name] = _extract_id_schema(index)
+
+        if not per_modality:
+            return {}
+
+        base = dict(next(iter(per_modality.values())))
+        base["full_id_separator"] = "/"
+
+        overrides: dict[str, dict[str, Any]] = {}
+        for mod_name, mod_schema in per_modality.items():
+            mod_with_separator = dict(mod_schema)
+            mod_with_separator["full_id_separator"] = "/"
+            if mod_with_separator != base:
+                overrides[mod_name] = mod_with_separator
+
+        if overrides:
+            base["modalities"] = overrides
+        return base
+
     def get_modality_index(self, modality_name: str) -> dict[str, Any]:
         """Return the cached ds-crawler index for a modality."""
         if modality_name in self._modalities:

{euler_loading-2.2.2 → euler_loading-2.4.0}/euler_loading/preprocessing.py

@@ -17,6 +17,7 @@ except ImportError:  # pragma: no cover - exercised in CPU-only environments
 
 __all__ = [
     "FieldSpec",
+    "MaskedValueOverride",
     "Resize",
     "Crop",
     "SamplePreprocessor",
@@ -310,6 +311,111 @@ class Crop:
         return top, left, target_h, target_w
 
 
+@dataclass(frozen=True)
+class MaskedValueOverride:
+    """Set values in one sample field wherever a boolean mask field is true."""
+
+    target_key: str
+    mask_key: str
+    value: Any
+    invert_mask: bool = False
+    copy: bool = True
+    ignore_missing: bool = False
+
+    @classmethod
+    def from_config(cls, cfg: Mapping[str, Any]) -> "MaskedValueOverride":
+        if not isinstance(cfg, Mapping):
+            raise TypeError(
+                "MaskedValueOverride config must be a mapping, "
+                f"got {type(cfg).__name__}."
+            )
+        return cls(
+            target_key=str(cfg.get("target_key", cfg.get("target", ""))),
+            mask_key=str(cfg.get("mask_key", cfg.get("mask", ""))),
+            value=cfg.get("value"),
+            invert_mask=bool(cfg.get("invert_mask", False)),
+            copy=bool(cfg.get("copy", True)),
+            ignore_missing=bool(cfg.get("ignore_missing", False)),
+        )
+
+    def __post_init__(self) -> None:
+        if not self.target_key:
+            raise ValueError("MaskedValueOverride requires target_key.")
+        if not self.mask_key:
+            raise ValueError("MaskedValueOverride requires mask_key.")
+
+    def describe(self) -> str:
+        parts = [
+            f"masked_value_override(target={self.target_key}",
+            f"mask={self.mask_key}",
+            f"value={self.value!r}",
+        ]
+        if self.invert_mask:
+            parts.append("invert_mask=True")
+        if not self.copy:
+            parts.append("copy=False")
+        if self.ignore_missing:
+            parts.append("ignore_missing=True")
+        return ", ".join(parts) + ")"
+
+    def _missing(self, key: str) -> None:
+        if self.ignore_missing:
+            return
+        raise KeyError(f"MaskedValueOverride missing sample key: {key!r}")
+
+    def __call__(self, sample: dict[str, Any]) -> dict[str, Any]:
+        if self.target_key not in sample:
+            self._missing(self.target_key)
+            return sample
+        if self.mask_key not in sample:
+            self._missing(self.mask_key)
+            return sample
+
+        target = sample[self.target_key]
+        mask = sample[self.mask_key]
+
+        processed = dict(sample) if self.copy else sample
+
+        if _is_torch_tensor(target):
+            if _is_torch_tensor(mask):
+                mask_tensor = mask.to(device=target.device, dtype=torch.bool)
+            else:
+                mask_tensor = torch.as_tensor(
+                    mask,
+                    device=target.device,
+                    dtype=torch.bool,
+                )
+            if self.invert_mask:
+                mask_tensor = ~mask_tensor
+            if tuple(mask_tensor.shape) != tuple(target.shape):
+                mask_tensor = torch.broadcast_to(mask_tensor, target.shape)
+            value_tensor = torch.as_tensor(
+                self.value,
+                device=target.device,
+                dtype=target.dtype,
+            )
+            result = target.clone() if self.copy else target
+            result[mask_tensor] = value_tensor
+            processed[self.target_key] = result
+            return processed
+
+        if isinstance(target, np.ndarray):
+            mask_array = np.asarray(mask, dtype=np.bool_)
+            if self.invert_mask:
+                mask_array = ~mask_array
+            if tuple(mask_array.shape) != tuple(target.shape):
+                mask_array = np.broadcast_to(mask_array, target.shape)
+            result = np.array(target, copy=self.copy)
+            result[mask_array] = self.value
+            processed[self.target_key] = result
+            return processed
+
+        raise TypeError(
+            f"MaskedValueOverride target {self.target_key!r} must be a torch.Tensor "
+            f"or numpy.ndarray, got {type(target).__name__}."
+        )
+
+
 def infer_field_spec(name: str, value: Any | None = None) -> FieldSpec | None:
     """Best-effort field inference for common modality names."""
 

{euler_loading-2.2.2 → euler_loading-2.4.0}/package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "l-more",
+  "name": "loading-better-save-outputs",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {}

{euler_loading-2.2.2 → euler_loading-2.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "euler-loading"
-version = "2.2.2"
+version = "2.4.0"
 description = "Multi-modal PyTorch dataloader using ds-crawler indices"
 requires-python = ">=3.9"
 dependencies = [

euler_loading-2.4.0/tests/test_id_schema.py

@@ -0,0 +1,116 @@
+"""Tests for MultiModalDataset.describe_id_schema."""
+
+from __future__ import annotations
+
+from typing import Any
+from unittest.mock import patch
+
+from euler_loading import Modality, MultiModalDataset
+
+from .conftest import dummy_loader
+
+
+def _flat_index(
+    file_ids: list[str],
+    *,
+    indexing: dict[str, Any] | None = None,
+    legacy_join_char: str | None = None,
+) -> dict[str, Any]:
+    index: dict[str, Any] = {
+        "name": "test-ds",
+        "type": "depth",
+        "euler_train": {"used_as": "input", "modality_type": "depth"},
+        "dataset": {
+            "files": [
+                {
+                    "id": fid,
+                    "path": f"scene/{fid}.png",
+                    "path_properties": {},
+                    "basename_properties": {},
+                }
+                for fid in file_ids
+            ]
+        },
+    }
+    if indexing is not None:
+        index["indexing"] = indexing
+    if legacy_join_char is not None:
+        index["id_regex_join_char"] = legacy_join_char
+    return index
+
+
+class TestDescribeIdSchema:
+    def test_modern_indexing_block(self):
+        index = _flat_index(
+            ["f001"],
+            indexing={
+                "id": {"regex": "^(.+)$", "join_char": "+"},
+                "hierarchy": {"separator": "-"},
+            },
+        )
+        with patch(
+            "euler_loading.dataset.index_dataset_from_path",
+            return_value=index,
+        ):
+            ds = MultiModalDataset(
+                modalities={"depth": Modality("/data/depth", loader=dummy_loader)}
+            )
+        schema = ds.describe_id_schema()
+        assert schema["hierarchy_separator"] == "-"
+        assert schema["id_join_char"] == "+"
+        assert schema["full_id_separator"] == "/"
+        assert schema["id_regex"] == "^(.+)$"
+
+    def test_legacy_top_level_join_char(self):
+        index = _flat_index(["f001"], legacy_join_char="|")
+        with patch(
+            "euler_loading.dataset.index_dataset_from_path",
+            return_value=index,
+        ):
+            ds = MultiModalDataset(
+                modalities={"depth": Modality("/data/depth", loader=dummy_loader)}
+            )
+        schema = ds.describe_id_schema()
+        assert schema["id_join_char"] == "|"
+        assert schema["hierarchy_separator"] == "-"
+
+    def test_missing_indexing_falls_back_to_defaults(self):
+        index = _flat_index(["f001"])
+        with patch(
+            "euler_loading.dataset.index_dataset_from_path",
+            return_value=index,
+        ):
+            ds = MultiModalDataset(
+                modalities={"depth": Modality("/data/depth", loader=dummy_loader)}
+            )
+        schema = ds.describe_id_schema()
+        assert schema["hierarchy_separator"] == "-"
+        assert schema["id_join_char"] == "+"
+        assert schema["full_id_separator"] == "/"
+
+    def test_divergent_modalities_record_overrides(self):
+        rgb_index = _flat_index(
+            ["f001"],
+            indexing={"id": {"join_char": "+"}, "hierarchy": {"separator": "-"}},
+        )
+        depth_index = _flat_index(
+            ["f001"],
+            indexing={"id": {"join_char": "|"}, "hierarchy": {"separator": "-"}},
+        )
+
+        def mock_index(path: str, **_: Any) -> dict[str, Any]:
+            return rgb_index if "rgb" in path else depth_index
+
+        with patch(
+            "euler_loading.dataset.index_dataset_from_path",
+            side_effect=mock_index,
+        ):
+            ds = MultiModalDataset(
+                modalities={
+                    "rgb": Modality("/data/rgb", loader=dummy_loader),
+                    "depth": Modality("/data/depth", loader=dummy_loader),
+                }
+            )
+        schema = ds.describe_id_schema()
+        assert "modalities" in schema
+        assert schema["modalities"]["depth"]["id_join_char"] == "|"

{euler_loading-2.2.2 → euler_loading-2.4.0}/tests/test_preprocessing.py

@@ -9,7 +9,7 @@ import numpy as np
 import pytest
 import torch
 
-from euler_loading import Modality, MultiModalDataset
+from euler_loading import MaskedValueOverride, Modality, MultiModalDataset
 from euler_loading.preprocessing import SamplePreprocessor
 
 
@@ -143,6 +143,63 @@ class TestSamplePreprocessorNumpy:
         assert np.allclose(processed["intrinsics"], expected_intrinsics)
 
 
+class TestMaskedValueOverride:
+    def test_overrides_torch_target_with_broadcast_mask(self):
+        depth = torch.tensor(
+            [
+                [[1.0, 2.0], [3.0, 4.0]],
+                [[5.0, 6.0], [7.0, 8.0]],
+            ],
+            dtype=torch.float32,
+        )
+        sky_mask = torch.tensor([[[True, False], [False, True]]])
+
+        transform = MaskedValueOverride(
+            target_key="depth",
+            mask_key="sky_mask",
+            value=0.5,
+        )
+        processed = transform({"depth": depth, "sky_mask": sky_mask})
+
+        expected = torch.tensor(
+            [
+                [[0.5, 2.0], [3.0, 0.5]],
+                [[0.5, 6.0], [7.0, 0.5]],
+            ],
+            dtype=torch.float32,
+        )
+        assert torch.equal(processed["depth"], expected)
+        assert torch.equal(
+            depth,
+            torch.tensor(
+                [
+                    [[1.0, 2.0], [3.0, 4.0]],
+                    [[5.0, 6.0], [7.0, 8.0]],
+                ],
+                dtype=torch.float32,
+            ),
+        )
+
+    def test_overrides_numpy_target_with_inverted_mask(self):
+        depth = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)
+        valid_mask = np.array([[True, False], [False, True]], dtype=np.bool_)
+
+        transform = MaskedValueOverride(
+            target_key="depth",
+            mask_key="valid_mask",
+            value=9.0,
+            invert_mask=True,
+        )
+        processed = transform({"depth": depth, "valid_mask": valid_mask})
+
+        expected = np.array([[1.0, 9.0], [9.0, 4.0]], dtype=np.float32)
+        assert np.array_equal(processed["depth"], expected)
+        assert np.array_equal(
+            depth,
+            np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32),
+        )
+
+
 class TestDatasetTransformBinding:
     def test_dataset_binds_modality_types_into_preprocessor(self):
         rgb_index = _flat_index("rgb", ["f001"])