euler-loading 2.10.0__tar.gz → 2.11.0__tar.gz

{euler_loading-2.10.0 → euler_loading-2.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: euler-loading
-Version: 2.10.0
+Version: 2.11.0
 Summary: Multi-modal PyTorch dataloader using ds-crawler indices
 Requires-Python: >=3.9
 Requires-Dist: ds-crawler>=2.3.0

{euler_loading-2.10.0 → euler_loading-2.11.0}/README.md

@@ -193,6 +193,17 @@ PyTorch `Dataset`. On construction it:
 | `hierarchical_modalities` | `dict[str, Modality] \| None` | Optional modalities whose files live at intermediate hierarchy levels (e.g. per-scene intrinsics). These do **not** participate in ID intersection. Each sample will contain a dict `{file_id: loaded_result}` with all files at or above the sample's hierarchy level. Results are cached so shared files are parsed only once. |
 | `transforms` | `list[Callable[[dict], dict]] \| None` | Applied in order after loading. Each receives and returns the full sample dict. |
 
+For hierarchical modalities, the ds-crawler file ID is the key in
+`{file_id: loaded_result}`. If files at different ancestor levels use the same
+ID, the deepest matching file wins. This gives calibration files natural
+inheritance semantics: a root-level `intrinsics` file can apply to every sample,
+while a scene- or camera-level `intrinsics` file with the same ID overrides it
+for descendants. Use ds-crawler's `indexing.id.override` when the physical file
+name is incidental, such as `calib.json` or a UUID, and you want a stable
+semantic key like `intrinsics`, `extrinsics`, or `calibration`. Do not use an
+override when multiple distinct files at the same hierarchy level should all be
+returned.
+
 #### Sample dict
 
 `dataset[i]` returns:
@@ -361,6 +372,11 @@ Each logical modality must have its own ds-crawler index. Usually that means one
 Files across regular modalities are matched by IDs from those indexes, so **the indexed hierarchy and naming conventions must be consistent** across modalities up to modality-specific parts captured in the config.
 
 Calibration files or other per-scene/per-sequence metadata can be loaded via `hierarchical_modalities`. These files are matched to samples based on their position in the hierarchy — all files at or above a sample's hierarchy level are included and cached for efficiency.
+Root-level calibration files, for example `calib.json` at the top of a shared
+zip, should be indexed by ds-crawler without a hierarchy block. euler-loading
+treats that root file as an ancestor of every sample. If the modality uses
+`collapse_single=True`, the loaded calibration value is returned directly as
+long as exactly one hierarchical file matches the sample.
 
 ## Layout-aware augmentation loading
 
@@ -484,6 +500,7 @@ All built-in loaders accept both filesystem paths (`str`) and in-memory buffers
 | `reference_rgb` | Clear-weather reference RGB image as float32, normalised to [0, 1] |
 | `semantic_segmentation` | Single-channel Cityscapes `labelIds` or `labelTrainIds` PNG as class IDs |
 | `semantic_segmentation_color` | Cityscapes `labelColor` PNG as RGB uint8 labels |
+| `sky_mask` | Binary mask from Cityscapes labels; uses sky `trainId=10` for `labelTrainIds`, `labelId=23` for `labelIds`, or RGB `meta["sky_class"]` such as `[0, 0, 23]` |
 | `panoptic_segmentation` | COCO-style RGB panoptic PNG decoded to integer segment IDs |
 | `lidar_point_cloud` / `point_cloud` | Lidar `.bin` file as `(N, 6)` float64: `x, y, z, intensity, ring, timestamp` |
 | `sparse_depth` | Alias for MUSES lidar points for sparse-depth style workflows |

{euler_loading-2.10.0 → euler_loading-2.11.0}/euler_loading/loaders/cpu/muses.py

@@ -12,6 +12,7 @@ Return types
   containing raw Cityscapes ``labelIds`` or ``labelTrainIds``.
 - **semantic_segmentation_color** -- ``np.ndarray`` of shape ``(H, W, 3)``
   uint8 containing Cityscapes RGB colours.
+- **sky_mask** -- ``np.ndarray`` of shape ``(H, W)`` bool.
 - **panoptic_segmentation** -- ``np.ndarray`` of shape ``(H, W)`` int64
   containing COCO-style panoptic segment IDs decoded from RGB.
 - **lidar_point_cloud** / **sparse_depth** -- ``np.ndarray`` of shape
@@ -35,6 +36,8 @@ from PIL import Image
 from euler_loading.loaders._annotations import modality_meta
 
 _LIDAR_COLUMNS = ["x", "y", "z", "intensity", "ring", "timestamp"]
+_SKY_LABEL_ID = 23
+_SKY_TRAIN_ID = 10
 
 
 def _load_image_rgb(path: Union[str, BinaryIO]) -> np.ndarray:
@@ -56,6 +59,59 @@ def _load_single_channel_labels(path: Union[str, BinaryIO]) -> np.ndarray:
     return arr
 
 
+def _select_sky_class_id(
+    path: Union[str, BinaryIO],
+    meta: Mapping[str, Any] | None,
+    attributes: Mapping[str, Any] | None,
+) -> int:
+    for source in (attributes, meta):
+        if not isinstance(source, Mapping):
+            continue
+        value = source.get("sky_class_id")
+        if isinstance(value, (int, np.integer)):
+            return int(value)
+
+    name = str(getattr(path, "name", path)).lower()
+    if "labelids" in name:
+        return _SKY_LABEL_ID
+    return _SKY_TRAIN_ID
+
+
+def _select_sky_class_color(
+    meta: Mapping[str, Any] | None,
+    attributes: Mapping[str, Any] | None,
+) -> np.ndarray | None:
+    for source in (attributes, meta):
+        if not isinstance(source, Mapping):
+            continue
+        value = source.get("sky_class")
+        if value is None:
+            value = source.get("sky_mask")
+        if value is None:
+            continue
+
+        color = np.asarray(value, dtype=np.uint8)
+        if color.shape != (3,):
+            raise ValueError("MUSES sky_class must be a 3-element RGB value")
+        return color
+    return None
+
+
+def _load_sky_mask_array(
+    path: Union[str, BinaryIO],
+    meta: Mapping[str, Any] | None,
+    attributes: Mapping[str, Any] | None,
+) -> np.ndarray:
+    color = _select_sky_class_color(meta, attributes)
+    if color is not None:
+        with Image.open(path) as image:
+            arr = np.array(image.convert("RGB"), dtype=np.uint8)
+        return np.all(arr == color, axis=-1)
+
+    labels = _load_single_channel_labels(path)
+    return labels == _select_sky_class_id(path, meta, attributes)
+
+
 def _load_rgb_uint8(path: Union[str, BinaryIO]) -> np.ndarray:
     """Load an RGB image without normalisation."""
     with Image.open(path) as image:
@@ -246,6 +302,29 @@ def semantic_segmentation(path: Union[str, BinaryIO], meta: dict[str, Any] | Non
     return _load_single_channel_labels(path)
 
 
+@modality_meta(
+    modality_type="sky_mask",
+    dtype="bool",
+    shape="HW",
+    file_formats=[".png"],
+    meta={
+        "encoding": "single_channel",
+        "sky_class": [0, 0, _SKY_LABEL_ID],
+        "sky_label_id": _SKY_LABEL_ID,
+        "sky_train_id": _SKY_TRAIN_ID,
+    },
+)
+def sky_mask(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None, *, attributes: dict[str, Any] | None = None) -> np.ndarray:
+    """Load a MUSES sky mask as an ``(H, W)`` bool array.
+
+    Reads a single-channel Cityscapes ``labelIds`` / ``labelTrainIds`` PNG, or
+    an RGB-encoded class image when ``sky_class`` is provided. Override the
+    default ID with ``sky_class_id``, or set ``sky_class`` to an RGB value such
+    as ``[0, 0, 23]``.
+    """
+    return _load_sky_mask_array(path, meta, attributes)
+
+
 @modality_meta(
     modality_type="semantic_segmentation_color",
     dtype="uint8",

{euler_loading-2.10.0 → euler_loading-2.11.0}/euler_loading/loaders/generate/loaders.json

@@ -396,6 +396,32 @@
                         "shape": "CHW"
                     }
                 },
+                {
+                    "type": "sky_mask",
+                    "function": "sky_mask",
+                    "hierarchical": false,
+                    "dtype": "bool",
+                    "access": "key",
+                    "file_formats": [
+                        ".png"
+                    ],
+                    "meta": {
+                        "encoding": "single_channel",
+                        "sky_class": [
+                            0,
+                            0,
+                            23
+                        ],
+                        "sky_label_id": 23,
+                        "sky_train_id": 10
+                    },
+                    "cpu": {
+                        "shape": "HW"
+                    },
+                    "gpu": {
+                        "shape": "1HW"
+                    }
+                },
                 {
                     "type": "sparse_depth",
                     "function": "sparse_depth",

{euler_loading-2.10.0 → euler_loading-2.11.0}/euler_loading/loaders/gpu/muses.py

@@ -12,6 +12,7 @@ Return types
   containing raw Cityscapes ``labelIds`` or ``labelTrainIds``.
 - **semantic_segmentation_color** -- ``torch.ByteTensor`` of shape
   ``(3, H, W)`` containing Cityscapes RGB colours.
+- **sky_mask** -- ``torch.BoolTensor`` of shape ``(1, H, W)``.
 - **panoptic_segmentation** -- ``torch.LongTensor`` of shape ``(1, H, W)``
   containing COCO-style panoptic segment IDs decoded from RGB.
 - **lidar_point_cloud** / **sparse_depth** -- ``torch.DoubleTensor`` of shape
@@ -36,6 +37,8 @@ from PIL import Image
 from euler_loading.loaders._annotations import modality_meta
 
 _LIDAR_COLUMNS = ["x", "y", "z", "intensity", "ring", "timestamp"]
+_SKY_LABEL_ID = 23
+_SKY_TRAIN_ID = 10
 
 
 def _load_image_rgb(path: Union[str, BinaryIO]) -> np.ndarray:
@@ -57,6 +60,59 @@ def _load_single_channel_labels(path: Union[str, BinaryIO]) -> np.ndarray:
     return arr
 
 
+def _select_sky_class_id(
+    path: Union[str, BinaryIO],
+    meta: Mapping[str, Any] | None,
+    attributes: Mapping[str, Any] | None,
+) -> int:
+    for source in (attributes, meta):
+        if not isinstance(source, Mapping):
+            continue
+        value = source.get("sky_class_id")
+        if isinstance(value, (int, np.integer)):
+            return int(value)
+
+    name = str(getattr(path, "name", path)).lower()
+    if "labelids" in name:
+        return _SKY_LABEL_ID
+    return _SKY_TRAIN_ID
+
+
+def _select_sky_class_color(
+    meta: Mapping[str, Any] | None,
+    attributes: Mapping[str, Any] | None,
+) -> np.ndarray | None:
+    for source in (attributes, meta):
+        if not isinstance(source, Mapping):
+            continue
+        value = source.get("sky_class")
+        if value is None:
+            value = source.get("sky_mask")
+        if value is None:
+            continue
+
+        color = np.asarray(value, dtype=np.uint8)
+        if color.shape != (3,):
+            raise ValueError("MUSES sky_class must be a 3-element RGB value")
+        return color
+    return None
+
+
+def _load_sky_mask_array(
+    path: Union[str, BinaryIO],
+    meta: Mapping[str, Any] | None,
+    attributes: Mapping[str, Any] | None,
+) -> np.ndarray:
+    color = _select_sky_class_color(meta, attributes)
+    if color is not None:
+        with Image.open(path) as image:
+            arr = np.array(image.convert("RGB"), dtype=np.uint8)
+        return np.all(arr == color, axis=-1)
+
+    labels = _load_single_channel_labels(path)
+    return labels == _select_sky_class_id(path, meta, attributes)
+
+
 def _load_rgb_uint8(path: Union[str, BinaryIO]) -> np.ndarray:
     """Load an RGB image without normalisation."""
     with Image.open(path) as image:
@@ -250,6 +306,30 @@ def semantic_segmentation(path: Union[str, BinaryIO], meta: dict[str, Any] | Non
     return torch.from_numpy(arr).unsqueeze(0).contiguous()
 
 
+@modality_meta(
+    modality_type="sky_mask",
+    dtype="bool",
+    shape="1HW",
+    file_formats=[".png"],
+    meta={
+        "encoding": "single_channel",
+        "sky_class": [0, 0, _SKY_LABEL_ID],
+        "sky_label_id": _SKY_LABEL_ID,
+        "sky_train_id": _SKY_TRAIN_ID,
+    },
+)
+def sky_mask(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None, *, attributes: dict[str, Any] | None = None) -> torch.Tensor:
+    """Load a MUSES sky mask as a ``(1, H, W)`` bool tensor.
+
+    Reads a single-channel Cityscapes ``labelIds`` / ``labelTrainIds`` PNG, or
+    an RGB-encoded class image when ``sky_class`` is provided. Override the
+    default ID with ``sky_class_id``, or set ``sky_class`` to an RGB value such
+    as ``[0, 0, 23]``.
+    """
+    mask = _load_sky_mask_array(path, meta, attributes)
+    return torch.from_numpy(mask).unsqueeze(0).contiguous()
+
+
 @modality_meta(
     modality_type="semantic_segmentation_color",
     dtype="uint8",

{euler_loading-2.10.0 → euler_loading-2.11.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "euler-loading"
-version = "2.10.0"
+version = "2.11.0"
 description = "Multi-modal PyTorch dataloader using ds-crawler indices"
 requires-python = ">=3.9"
 dependencies = [

{euler_loading-2.10.0 → euler_loading-2.11.0}/tests/test_loaders.py

@@ -158,6 +158,7 @@ MUSES_LOADER_NAMES = [
     "reference_rgb",
     "semantic_segmentation",
     "semantic_segmentation_color",
+    "sky_mask",
     "panoptic_segmentation",
     "lidar_point_cloud",
     "point_cloud",
@@ -189,6 +190,33 @@ def muses_semantic_path(tmp_path):
     return str(path), arr
 
 
+@pytest.fixture()
+def muses_sky_mask_path(tmp_path):
+    arr = np.array([[0, 10], [10, 255]], dtype=np.uint8)
+    path = tmp_path / "REC0001_frame_000001_gt_labelTrainIds.png"
+    Image.fromarray(arr, mode="L").save(path)
+    return str(path), arr == 10
+
+
+@pytest.fixture()
+def muses_sky_label_ids_path(tmp_path):
+    arr = np.array([[0, 23], [23, 255]], dtype=np.uint8)
+    path = tmp_path / "REC0001_frame_000001_gt_labelIds.png"
+    Image.fromarray(arr, mode="L").save(path)
+    return str(path), arr == 23
+
+
+@pytest.fixture()
+def muses_sky_rgb_class_path(tmp_path):
+    arr = np.zeros((2, 2, 3), dtype=np.uint8)
+    arr[0, 1] = [0, 0, 23]
+    arr[1, 0] = [0, 0, 23]
+    arr[1, 1] = [0, 0, 7]
+    path = tmp_path / "REC0001_frame_000001_gt_labelIds_rgb.png"
+    Image.fromarray(arr, mode="RGB").save(path)
+    return str(path), np.all(arr == np.array([0, 0, 23], dtype=np.uint8), axis=-1)
+
+
 @pytest.fixture()
 def muses_semantic_color_path(tmp_path):
     arr = np.array(
@@ -320,6 +348,23 @@ class TestMUSESGPULoaders:
         assert result.shape == (1, 2, 2)
         assert torch.equal(result, torch.from_numpy(expected.astype(np.int64)).unsqueeze(0))
 
+    def test_sky_mask_uses_train_ids(self, muses_sky_mask_path):
+        path, expected = muses_sky_mask_path
+        result = gpu_muses.sky_mask(path)
+        assert result.dtype == torch.bool
+        assert result.shape == (1, 2, 2)
+        assert torch.equal(result, torch.from_numpy(expected).unsqueeze(0))
+
+    def test_sky_mask_uses_label_ids_from_filename(self, muses_sky_label_ids_path):
+        path, expected = muses_sky_label_ids_path
+        result = gpu_muses.sky_mask(path)
+        assert torch.equal(result, torch.from_numpy(expected).unsqueeze(0))
+
+    def test_sky_mask_uses_rgb_sky_class_meta(self, muses_sky_rgb_class_path):
+        path, expected = muses_sky_rgb_class_path
+        result = gpu_muses.sky_mask(path, meta={"sky_class": [0, 0, 23]})
+        assert torch.equal(result, torch.from_numpy(expected).unsqueeze(0))
+
     def test_semantic_color_is_chw_uint8(self, muses_semantic_color_path):
         path, expected = muses_semantic_color_path
         result = gpu_muses.semantic_segmentation_color(path)
@@ -393,6 +438,23 @@ class TestMUSESCPULoaders:
         assert result.shape == (2, 2)
         assert np.array_equal(result, expected.astype(np.int64))
 
+    def test_sky_mask_uses_train_ids(self, muses_sky_mask_path):
+        path, expected = muses_sky_mask_path
+        result = cpu_muses.sky_mask(path)
+        assert result.dtype == np.bool_
+        assert result.shape == (2, 2)
+        assert np.array_equal(result, expected)
+
+    def test_sky_mask_uses_label_ids_from_filename(self, muses_sky_label_ids_path):
+        path, expected = muses_sky_label_ids_path
+        result = cpu_muses.sky_mask(path)
+        assert np.array_equal(result, expected)
+
+    def test_sky_mask_uses_rgb_sky_class_meta(self, muses_sky_rgb_class_path):
+        path, expected = muses_sky_rgb_class_path
+        result = cpu_muses.sky_mask(path, meta={"sky_class": [0, 0, 23]})
+        assert np.array_equal(result, expected)
+
     def test_semantic_color_is_hwc_uint8(self, muses_semantic_color_path):
         path, expected = muses_semantic_color_path
         result = cpu_muses.semantic_segmentation_color(path)