euler-loading 2.3.0__tar.gz → 2.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {euler_loading-2.3.0 → euler_loading-2.5.0}/PKG-INFO +1 -1
  2. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/dataset.py +82 -0
  3. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/cpu/generic.py +72 -0
  4. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/generate/loaders.json +68 -0
  5. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/gpu/generic.py +60 -0
  6. {euler_loading-2.3.0 → euler_loading-2.5.0}/package-lock.json +1 -1
  7. {euler_loading-2.3.0 → euler_loading-2.5.0}/pyproject.toml +1 -1
  8. euler_loading-2.5.0/tests/test_id_schema.py +116 -0
  9. {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/test_loaders.py +186 -2
  10. {euler_loading-2.3.0 → euler_loading-2.5.0}/.github/workflows/workflow.yml +0 -0
  11. {euler_loading-2.3.0 → euler_loading-2.5.0}/.gitignore +0 -0
  12. {euler_loading-2.3.0 → euler_loading-2.5.0}/README.md +0 -0
  13. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/__init__.py +0 -0
  14. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/_dataset_contract.py +0 -0
  15. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/_ds_crawler_utils.py +0 -0
  16. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/_metadata.py +0 -0
  17. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/_resolution.py +0 -0
  18. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/_writing.py +0 -0
  19. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/indexing.py +0 -0
  20. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/__init__.py +0 -0
  21. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/_annotations.py +0 -0
  22. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/_writer_utils.py +0 -0
  23. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/contracts.py +0 -0
  24. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/cpu/__init__.py +0 -0
  25. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/cpu/generic_dense_depth.py +0 -0
  26. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/cpu/real_drive_sim.py +0 -0
  27. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/cpu/vkitti2.py +0 -0
  28. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/generate/__init__.py +0 -0
  29. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/generate/__main__.py +0 -0
  30. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/generic.py +0 -0
  31. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/gpu/__init__.py +0 -0
  32. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/gpu/generic_dense_depth.py +0 -0
  33. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/gpu/real_drive_sim.py +0 -0
  34. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/gpu/vkitti2.py +0 -0
  35. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/real_drive_sim.py +0 -0
  36. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/vkitti2.py +0 -0
  37. {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/preprocessing.py +0 -0
  38. {euler_loading-2.3.0 → euler_loading-2.5.0}/example.py +0 -0
  39. {euler_loading-2.3.0 → euler_loading-2.5.0}/sample_rds.py +0 -0
  40. {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/__init__.py +0 -0
  41. {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/conftest.py +0 -0
  42. {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/example_rds_calib.json +0 -0
  43. {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/test_dataset.py +0 -0
  44. {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/test_indexing.py +0 -0
  45. {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/test_preprocessing.py +0 -0
  46. {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/test_real_dataset.py +0 -0
  47. {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/test_writing.py +0 -0
  48. {euler_loading-2.3.0 → euler_loading-2.5.0}/vkitti_cpu_example_output.json +0 -0
  49. {euler_loading-2.3.0 → euler_loading-2.5.0}/vkitti_gpu_example_output.json +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: euler-loading
3
- Version: 2.3.0
3
+ Version: 2.5.0
4
4
  Summary: Multi-modal PyTorch dataloader using ds-crawler indices
5
5
  Requires-Python: >=3.9
6
6
  Requires-Dist: ds-crawler
@@ -118,6 +118,36 @@ def _get_index_meta(index_output: Mapping[str, Any]) -> dict[str, Any] | None:
118
118
  return None
119
119
 
120
120
 
121
+ def _extract_id_schema(index_output: Mapping[str, Any]) -> dict[str, Any]:
122
+ """Pull id/hierarchy separators from a ds-crawler output payload.
123
+
124
+ Supports both modern (``indexing.id.join_char``) and legacy
125
+ (top-level ``id_regex_join_char``) layouts. Missing keys fall back to
126
+ ds-crawler defaults so the returned dict is always usable.
127
+ """
128
+ indexing = index_output.get("indexing")
129
+ indexing_map = indexing if isinstance(indexing, Mapping) else {}
130
+ id_cfg = indexing_map.get("id")
131
+ id_map = id_cfg if isinstance(id_cfg, Mapping) else {}
132
+ hierarchy_cfg = indexing_map.get("hierarchy")
133
+ hierarchy_map = hierarchy_cfg if isinstance(hierarchy_cfg, Mapping) else {}
134
+
135
+ id_join_char = id_map.get("join_char") or index_output.get("id_regex_join_char") or "+"
136
+ hierarchy_separator = hierarchy_map.get("separator") or "-"
137
+ id_regex = id_map.get("regex") or index_output.get("id_regex")
138
+ hierarchy_regex = hierarchy_map.get("regex") or index_output.get("hierarchy_regex")
139
+
140
+ schema: dict[str, Any] = {
141
+ "hierarchy_separator": hierarchy_separator,
142
+ "id_join_char": id_join_char,
143
+ }
144
+ if id_regex:
145
+ schema["id_regex"] = id_regex
146
+ if hierarchy_regex:
147
+ schema["hierarchy_regex"] = hierarchy_regex
148
+ return schema
149
+
150
+
121
151
  #TODO: might want to add slots=True in a 3.10+ only codebase
122
152
  @dataclass(frozen=True)
123
153
  class Modality:
@@ -488,6 +518,58 @@ class MultiModalDataset(_BaseDataset):
488
518
  meta = _get_index_meta(index)
489
519
  return meta or {}
490
520
 
521
+ def describe_id_schema(self) -> dict[str, Any]:
522
+ """Return the id-construction schema used by this dataset.
523
+
524
+ Downstream consumers that want to split or match ``sample["full_id"]``
525
+ / ``sample["id"]`` values back to ds-crawler fields need to know the
526
+ separators used to build them. This method exposes that schema so it
527
+ can be embedded in experiment manifests.
528
+
529
+ Schema keys:
530
+ - ``hierarchy_separator``: the character between the named
531
+ capture group *name* and its *value* within a single id part
532
+ (ds-crawler ``indexing.hierarchy.separator``, typically ``"-"``).
533
+ - ``id_join_char``: the character between id parts (ds-crawler
534
+ ``indexing.id.join_char`` or legacy ``id_regex_join_char``,
535
+ default ``"+"``).
536
+ - ``full_id_separator``: the character between hierarchy levels
537
+ and the file id in ``sample["full_id"]`` (currently ``"/"``).
538
+ - ``id_regex`` / ``hierarchy_regex``: the raw regexes, when
539
+ recorded in the ds-crawler output.
540
+ - ``modalities``: per-modality overrides populated only when
541
+ modalities disagree on the above.
542
+
543
+ Example::
544
+
545
+ {
546
+ "hierarchy_separator": "-",
547
+ "id_join_char": "+",
548
+ "full_id_separator": "/",
549
+ "id_regex": "...",
550
+ }
551
+ """
552
+ per_modality: dict[str, dict[str, Any]] = {}
553
+ for name, index in self._index_outputs.items():
554
+ per_modality[name] = _extract_id_schema(index)
555
+
556
+ if not per_modality:
557
+ return {}
558
+
559
+ base = dict(next(iter(per_modality.values())))
560
+ base["full_id_separator"] = "/"
561
+
562
+ overrides: dict[str, dict[str, Any]] = {}
563
+ for mod_name, mod_schema in per_modality.items():
564
+ mod_with_separator = dict(mod_schema)
565
+ mod_with_separator["full_id_separator"] = "/"
566
+ if mod_with_separator != base:
567
+ overrides[mod_name] = mod_with_separator
568
+
569
+ if overrides:
570
+ base["modalities"] = overrides
571
+ return base
572
+
491
573
  def get_modality_index(self, modality_name: str) -> dict[str, Any]:
492
574
  """Return the cached ds-crawler index for a modality."""
493
575
  if modality_name in self._modalities:
@@ -10,6 +10,8 @@ Supported file extensions:
10
10
 
11
11
  Return types
12
12
  ------------
13
+ - **map_2d** -- ``np.ndarray`` of shape ``(H, W)`` float32.
14
+ - **map_3d** -- ``np.ndarray`` of shape ``(H, W, C)`` float32.
13
15
  - **spherical_map** -- ``np.ndarray`` of shape ``(H, W, C)`` float32.
14
16
  - **intrinsics** -- ``np.ndarray`` of shape ``(3, 3)`` float32.
15
17
  - **sh_coeffs** -- ``np.ndarray`` of shape ``(N, 3)`` float32.
@@ -19,6 +21,8 @@ Usage::
19
21
  from euler_loading.loaders.cpu import generic
20
22
  from euler_loading import Modality
21
23
 
24
+ Modality("/data/my_dataset/scattering_coefficient", loader=generic.map_2d)
25
+ Modality("/data/my_dataset/atmospheric_light", loader=generic.map_3d)
22
26
  Modality("/data/my_dataset/spherical_map", loader=generic.spherical_map)
23
27
  Modality("/data/my_dataset/intrinsics", loader=generic.intrinsics)
24
28
  Modality("/data/my_dataset/sh_coeffs", loader=generic.sh_coeffs)
@@ -36,6 +40,7 @@ from euler_loading.loaders._writer_utils import (
36
40
  ensure_parent,
37
41
  get_target_name,
38
42
  mark_stream_supported,
43
+ to_hw,
39
44
  to_numpy,
40
45
  )
41
46
 
@@ -85,6 +90,40 @@ def _write_numpy(path: Union[str, BinaryIO], value: Any) -> None:
85
90
  # ---------------------------------------------------------------------------
86
91
 
87
92
 
93
+ @modality_meta(
94
+ modality_type="map_2d",
95
+ dtype="float32",
96
+ shape="HW",
97
+ file_formats=[".npy", ".npz"],
98
+ )
99
+ def map_2d(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> np.ndarray:
100
+ """Load an arbitrary 2D map as an ``(H, W)`` float32 array.
101
+
102
+ Suitable for any single-channel dense quantity (e.g. scattering
103
+ coefficient, attenuation, opacity) where no dataset-specific decoding
104
+ is required.
105
+ """
106
+ return _load_numpy(path)
107
+
108
+
109
+ @modality_meta(
110
+ modality_type="map_3d",
111
+ dtype="float32",
112
+ shape="HWC",
113
+ file_formats=[".npy", ".npz"],
114
+ )
115
+ def map_3d(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> np.ndarray:
116
+ """Load an arbitrary 3D map as an ``(H, W, C)`` float32 array.
117
+
118
+ The file is expected to be stored in ``(C, H, W)`` layout and is
119
+ transposed to ``(H, W, C)`` for CPU-oriented processing. Suitable
120
+ for any per-pixel vector quantity (e.g. atmospheric light, surface
121
+ normals, flow).
122
+ """
123
+ arr = _load_numpy(path)
124
+ return np.transpose(arr, (1, 2, 0))
125
+
126
+
88
127
  @modality_meta(
89
128
  modality_type="spherical_map",
90
129
  dtype="float32",
@@ -145,6 +184,39 @@ def sh_coeffs(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) ->
145
184
  # ---------------------------------------------------------------------------
146
185
 
147
186
 
187
+ @mark_stream_supported
188
+ def write_map_2d(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
189
+ """Write a 2D map to NumPy formats based on extension.
190
+
191
+ Accepts ``(H, W)``, ``(1, H, W)``, or ``(H, W, 1)`` input and stores
192
+ the array in ``(H, W)`` layout.
193
+ """
194
+ arr = to_hw(value, name="map_2d")
195
+ _write_numpy(path, arr)
196
+
197
+
198
+ @mark_stream_supported
199
+ def write_map_3d(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
200
+ """Write a 3D map to NumPy formats based on extension.
201
+
202
+ Input is expected in ``(H, W, C)`` layout and is transposed to
203
+ ``(C, H, W)`` for storage.
204
+ """
205
+ ensure_parent(path)
206
+ ext = os.path.splitext(_get_name(path))[1].lower()
207
+ arr = to_numpy(value).astype(np.float32)
208
+ arr = np.transpose(arr, (2, 0, 1))
209
+
210
+ if ext == _NPY_EXTENSION:
211
+ np.save(path, arr)
212
+ return
213
+ if ext == _NPZ_EXTENSION:
214
+ np.savez_compressed(path, data=arr)
215
+ return
216
+
217
+ raise ValueError(f"Unsupported map_3d output extension: {ext}")
218
+
219
+
148
220
  @mark_stream_supported
149
221
  def write_spherical_map(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
150
222
  """Write spherical-map data to NumPy formats based on extension.
@@ -3,6 +3,74 @@
3
3
  {
4
4
  "name": "generic",
5
5
  "modalities": [
6
+ {
7
+ "type": "intrinsics",
8
+ "function": "intrinsics",
9
+ "hierarchical": false,
10
+ "dtype": "float32",
11
+ "access": "key",
12
+ "file_formats": [
13
+ ".npy",
14
+ ".npz"
15
+ ],
16
+ "cpu": {
17
+ "shape": "3x3"
18
+ },
19
+ "gpu": {
20
+ "shape": "3x3"
21
+ }
22
+ },
23
+ {
24
+ "type": "map_2d",
25
+ "function": "map_2d",
26
+ "hierarchical": false,
27
+ "dtype": "float32",
28
+ "access": "key",
29
+ "file_formats": [
30
+ ".npy",
31
+ ".npz"
32
+ ],
33
+ "cpu": {
34
+ "shape": "HW"
35
+ },
36
+ "gpu": {
37
+ "shape": "HW"
38
+ }
39
+ },
40
+ {
41
+ "type": "map_3d",
42
+ "function": "map_3d",
43
+ "hierarchical": false,
44
+ "dtype": "float32",
45
+ "access": "key",
46
+ "file_formats": [
47
+ ".npy",
48
+ ".npz"
49
+ ],
50
+ "cpu": {
51
+ "shape": "HWC"
52
+ },
53
+ "gpu": {
54
+ "shape": "CHW"
55
+ }
56
+ },
57
+ {
58
+ "type": "sh_coeffs",
59
+ "function": "sh_coeffs",
60
+ "hierarchical": false,
61
+ "dtype": "float32",
62
+ "access": "key",
63
+ "file_formats": [
64
+ ".npy",
65
+ ".npz"
66
+ ],
67
+ "cpu": {
68
+ "shape": "NC"
69
+ },
70
+ "gpu": {
71
+ "shape": "NC"
72
+ }
73
+ },
6
74
  {
7
75
  "type": "spherical_map",
8
76
  "function": "spherical_map",
@@ -10,6 +10,8 @@ Supported file extensions:
10
10
 
11
11
  Return types
12
12
  ------------
13
+ - **map_2d** -- ``torch.FloatTensor`` of shape ``(H, W)``.
14
+ - **map_3d** -- ``torch.FloatTensor`` of shape ``(C, H, W)``.
13
15
  - **spherical_map** -- ``torch.FloatTensor`` of shape ``(C, H, W)``.
14
16
  - **intrinsics** -- ``torch.FloatTensor`` of shape ``(3, 3)``.
15
17
  - **sh_coeffs** -- ``torch.FloatTensor`` of shape ``(N, 3)``.
@@ -19,6 +21,8 @@ Usage::
19
21
  from euler_loading.loaders.gpu import generic
20
22
  from euler_loading import Modality
21
23
 
24
+ Modality("/data/my_dataset/scattering_coefficient", loader=generic.map_2d)
25
+ Modality("/data/my_dataset/atmospheric_light", loader=generic.map_3d)
22
26
  Modality("/data/my_dataset/spherical_map", loader=generic.spherical_map)
23
27
  Modality("/data/my_dataset/intrinsics", loader=generic.intrinsics)
24
28
  Modality("/data/my_dataset/sh_coeffs", loader=generic.sh_coeffs)
@@ -37,6 +41,7 @@ from euler_loading.loaders._writer_utils import (
37
41
  ensure_parent,
38
42
  get_target_name,
39
43
  mark_stream_supported,
44
+ to_hw,
40
45
  to_numpy,
41
46
  )
42
47
 
@@ -86,6 +91,40 @@ def _write_numpy(path: Union[str, BinaryIO], value: Any) -> None:
86
91
  # ---------------------------------------------------------------------------
87
92
 
88
93
 
94
+ @modality_meta(
95
+ modality_type="map_2d",
96
+ dtype="float32",
97
+ shape="HW",
98
+ file_formats=[".npy", ".npz"],
99
+ )
100
+ def map_2d(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> torch.Tensor:
101
+ """Load an arbitrary 2D map as an ``(H, W)`` float32 tensor.
102
+
103
+ Suitable for any single-channel dense quantity (e.g. scattering
104
+ coefficient, attenuation, opacity) where no dataset-specific decoding
105
+ is required.
106
+ """
107
+ arr = _load_numpy(path)
108
+ return torch.from_numpy(arr).contiguous()
109
+
110
+
111
+ @modality_meta(
112
+ modality_type="map_3d",
113
+ dtype="float32",
114
+ shape="CHW",
115
+ file_formats=[".npy", ".npz"],
116
+ )
117
+ def map_3d(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> torch.Tensor:
118
+ """Load an arbitrary 3D map as a ``(C, H, W)`` float32 tensor.
119
+
120
+ The file is expected to already be stored in ``(C, H, W)`` layout,
121
+ matching the torch convention. Suitable for any per-pixel vector
122
+ quantity (e.g. atmospheric light, surface normals, flow).
123
+ """
124
+ arr = _load_numpy(path)
125
+ return torch.from_numpy(arr).contiguous()
126
+
127
+
89
128
  @modality_meta(
90
129
  modality_type="spherical_map",
91
130
  dtype="float32",
@@ -147,6 +186,27 @@ def sh_coeffs(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) ->
147
186
  # ---------------------------------------------------------------------------
148
187
 
149
188
 
189
+ @mark_stream_supported
190
+ def write_map_2d(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
191
+ """Write a 2D map to NumPy formats based on extension.
192
+
193
+ Accepts ``(H, W)``, ``(1, H, W)``, or ``(H, W, 1)`` input and stores
194
+ the array in ``(H, W)`` layout.
195
+ """
196
+ arr = to_hw(value, name="map_2d")
197
+ _write_numpy(path, arr)
198
+
199
+
200
+ @mark_stream_supported
201
+ def write_map_3d(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
202
+ """Write a 3D map to NumPy formats based on extension.
203
+
204
+ Input is expected in ``(C, H, W)`` layout (torch convention) and is
205
+ stored as-is.
206
+ """
207
+ _write_numpy(path, value)
208
+
209
+
150
210
  @mark_stream_supported
151
211
  def write_spherical_map(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
152
212
  """Write spherical-map data to NumPy formats based on extension."""
@@ -1,5 +1,5 @@
1
1
  {
2
- "name": "l-more",
2
+ "name": "loading-2D-map",
3
3
  "lockfileVersion": 3,
4
4
  "requires": true,
5
5
  "packages": {}
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "euler-loading"
3
- version = "2.3.0"
3
+ version = "2.5.0"
4
4
  description = "Multi-modal PyTorch dataloader using ds-crawler indices"
5
5
  requires-python = ">=3.9"
6
6
  dependencies = [
@@ -0,0 +1,116 @@
1
+ """Tests for MultiModalDataset.describe_id_schema."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+ from unittest.mock import patch
7
+
8
+ from euler_loading import Modality, MultiModalDataset
9
+
10
+ from .conftest import dummy_loader
11
+
12
+
13
+ def _flat_index(
14
+ file_ids: list[str],
15
+ *,
16
+ indexing: dict[str, Any] | None = None,
17
+ legacy_join_char: str | None = None,
18
+ ) -> dict[str, Any]:
19
+ index: dict[str, Any] = {
20
+ "name": "test-ds",
21
+ "type": "depth",
22
+ "euler_train": {"used_as": "input", "modality_type": "depth"},
23
+ "dataset": {
24
+ "files": [
25
+ {
26
+ "id": fid,
27
+ "path": f"scene/{fid}.png",
28
+ "path_properties": {},
29
+ "basename_properties": {},
30
+ }
31
+ for fid in file_ids
32
+ ]
33
+ },
34
+ }
35
+ if indexing is not None:
36
+ index["indexing"] = indexing
37
+ if legacy_join_char is not None:
38
+ index["id_regex_join_char"] = legacy_join_char
39
+ return index
40
+
41
+
42
+ class TestDescribeIdSchema:
43
+ def test_modern_indexing_block(self):
44
+ index = _flat_index(
45
+ ["f001"],
46
+ indexing={
47
+ "id": {"regex": "^(.+)$", "join_char": "+"},
48
+ "hierarchy": {"separator": "-"},
49
+ },
50
+ )
51
+ with patch(
52
+ "euler_loading.dataset.index_dataset_from_path",
53
+ return_value=index,
54
+ ):
55
+ ds = MultiModalDataset(
56
+ modalities={"depth": Modality("/data/depth", loader=dummy_loader)}
57
+ )
58
+ schema = ds.describe_id_schema()
59
+ assert schema["hierarchy_separator"] == "-"
60
+ assert schema["id_join_char"] == "+"
61
+ assert schema["full_id_separator"] == "/"
62
+ assert schema["id_regex"] == "^(.+)$"
63
+
64
+ def test_legacy_top_level_join_char(self):
65
+ index = _flat_index(["f001"], legacy_join_char="|")
66
+ with patch(
67
+ "euler_loading.dataset.index_dataset_from_path",
68
+ return_value=index,
69
+ ):
70
+ ds = MultiModalDataset(
71
+ modalities={"depth": Modality("/data/depth", loader=dummy_loader)}
72
+ )
73
+ schema = ds.describe_id_schema()
74
+ assert schema["id_join_char"] == "|"
75
+ assert schema["hierarchy_separator"] == "-"
76
+
77
+ def test_missing_indexing_falls_back_to_defaults(self):
78
+ index = _flat_index(["f001"])
79
+ with patch(
80
+ "euler_loading.dataset.index_dataset_from_path",
81
+ return_value=index,
82
+ ):
83
+ ds = MultiModalDataset(
84
+ modalities={"depth": Modality("/data/depth", loader=dummy_loader)}
85
+ )
86
+ schema = ds.describe_id_schema()
87
+ assert schema["hierarchy_separator"] == "-"
88
+ assert schema["id_join_char"] == "+"
89
+ assert schema["full_id_separator"] == "/"
90
+
91
+ def test_divergent_modalities_record_overrides(self):
92
+ rgb_index = _flat_index(
93
+ ["f001"],
94
+ indexing={"id": {"join_char": "+"}, "hierarchy": {"separator": "-"}},
95
+ )
96
+ depth_index = _flat_index(
97
+ ["f001"],
98
+ indexing={"id": {"join_char": "|"}, "hierarchy": {"separator": "-"}},
99
+ )
100
+
101
+ def mock_index(path: str, **_: Any) -> dict[str, Any]:
102
+ return rgb_index if "rgb" in path else depth_index
103
+
104
+ with patch(
105
+ "euler_loading.dataset.index_dataset_from_path",
106
+ side_effect=mock_index,
107
+ ):
108
+ ds = MultiModalDataset(
109
+ modalities={
110
+ "rgb": Modality("/data/rgb", loader=dummy_loader),
111
+ "depth": Modality("/data/depth", loader=dummy_loader),
112
+ }
113
+ )
114
+ schema = ds.describe_id_schema()
115
+ assert "modalities" in schema
116
+ assert schema["modalities"]["depth"]["id_join_char"] == "|"
@@ -485,8 +485,8 @@ class TestRDSWriters:
485
485
  # Generic spherical_map loader tests
486
486
  # ---------------------------------------------------------------------------
487
487
 
488
- GENERIC_LOADER_NAMES = ["spherical_map"]
489
- GENERIC_WRITER_NAMES = ["write_spherical_map"]
488
+ GENERIC_LOADER_NAMES = ["map_2d", "map_3d", "spherical_map"]
489
+ GENERIC_WRITER_NAMES = ["write_map_2d", "write_map_3d", "write_spherical_map"]
490
490
 
491
491
 
492
492
  @pytest.fixture()
@@ -638,3 +638,187 @@ class TestGenericBackwardCompat:
638
638
  top = generic_top.spherical_map(path)
639
639
  gpu = gpu_generic.spherical_map(path)
640
640
  assert torch.equal(top, gpu)
641
+
642
+
643
+ # ---------------------------------------------------------------------------
644
+ # Generic map_2d / map_3d loader tests
645
+ # ---------------------------------------------------------------------------
646
+
647
+
648
+ @pytest.fixture()
649
+ def map_2d_npy_path(tmp_path):
650
+ """Write a small (4, 5) float32 .npy file."""
651
+ arr = np.random.default_rng(0).random((4, 5), dtype=np.float32)
652
+ p = tmp_path / "map_2d.npy"
653
+ np.save(str(p), arr)
654
+ return str(p), arr
655
+
656
+
657
+ @pytest.fixture()
658
+ def map_2d_npz_path(tmp_path):
659
+ """Write a small (4, 5) float32 .npz file."""
660
+ arr = np.random.default_rng(1).random((4, 5), dtype=np.float32)
661
+ p = tmp_path / "map_2d.npz"
662
+ np.savez_compressed(str(p), data=arr)
663
+ return str(p), arr
664
+
665
+
666
+ @pytest.fixture()
667
+ def map_3d_npy_path(tmp_path):
668
+ """Write a small (2, 4, 5) float32 .npy file in CHW layout."""
669
+ arr = np.random.default_rng(2).random((2, 4, 5), dtype=np.float32)
670
+ p = tmp_path / "map_3d.npy"
671
+ np.save(str(p), arr)
672
+ return str(p), arr
673
+
674
+
675
+ @pytest.fixture()
676
+ def map_3d_npz_path(tmp_path):
677
+ """Write a small (2, 4, 5) float32 .npz file in CHW layout."""
678
+ arr = np.random.default_rng(3).random((2, 4, 5), dtype=np.float32)
679
+ p = tmp_path / "map_3d.npz"
680
+ np.savez_compressed(str(p), data=arr)
681
+ return str(p), arr
682
+
683
+
684
+ class TestGPUMap2DLoader:
685
+ """GPU ``map_2d`` loads a 2D map as an ``(H, W)`` float32 tensor."""
686
+
687
+ def test_npy_dtype_and_shape(self, map_2d_npy_path):
688
+ path, _ = map_2d_npy_path
689
+ result = gpu_generic.map_2d(path)
690
+ assert isinstance(result, torch.Tensor)
691
+ assert result.dtype == torch.float32
692
+ assert result.shape == (4, 5)
693
+
694
+ def test_npy_values_match(self, map_2d_npy_path):
695
+ path, expected = map_2d_npy_path
696
+ result = gpu_generic.map_2d(path)
697
+ assert torch.allclose(result, torch.from_numpy(expected))
698
+
699
+ def test_npz_dtype_and_shape(self, map_2d_npz_path):
700
+ path, _ = map_2d_npz_path
701
+ result = gpu_generic.map_2d(path)
702
+ assert result.dtype == torch.float32
703
+ assert result.shape == (4, 5)
704
+
705
+
706
+ class TestCPUMap2DLoader:
707
+ """CPU ``map_2d`` loads a 2D map as an ``(H, W)`` float32 array."""
708
+
709
+ def test_npy_dtype_and_shape(self, map_2d_npy_path):
710
+ path, _ = map_2d_npy_path
711
+ result = cpu_generic.map_2d(path)
712
+ assert isinstance(result, np.ndarray)
713
+ assert result.dtype == np.float32
714
+ assert result.shape == (4, 5)
715
+
716
+ def test_npy_values_match(self, map_2d_npy_path):
717
+ path, expected = map_2d_npy_path
718
+ result = cpu_generic.map_2d(path)
719
+ assert np.allclose(result, expected)
720
+
721
+ def test_npz_dtype_and_shape(self, map_2d_npz_path):
722
+ path, _ = map_2d_npz_path
723
+ result = cpu_generic.map_2d(path)
724
+ assert result.dtype == np.float32
725
+ assert result.shape == (4, 5)
726
+
727
+
728
+ class TestGPUMap3DLoader:
729
+ """GPU ``map_3d`` loads a 3D map as a ``(C, H, W)`` float32 tensor."""
730
+
731
+ def test_npy_dtype_and_shape(self, map_3d_npy_path):
732
+ path, _ = map_3d_npy_path
733
+ result = gpu_generic.map_3d(path)
734
+ assert isinstance(result, torch.Tensor)
735
+ assert result.dtype == torch.float32
736
+ assert result.shape == (2, 4, 5)
737
+
738
+ def test_npy_values_match(self, map_3d_npy_path):
739
+ path, expected = map_3d_npy_path
740
+ result = gpu_generic.map_3d(path)
741
+ assert torch.allclose(result, torch.from_numpy(expected))
742
+
743
+ def test_npz_dtype_and_shape(self, map_3d_npz_path):
744
+ path, _ = map_3d_npz_path
745
+ result = gpu_generic.map_3d(path)
746
+ assert result.shape == (2, 4, 5)
747
+
748
+
749
+ class TestCPUMap3DLoader:
750
+ """CPU ``map_3d`` transposes CHW-on-disk to ``(H, W, C)``."""
751
+
752
+ def test_npy_dtype_and_shape(self, map_3d_npy_path):
753
+ path, _ = map_3d_npy_path
754
+ result = cpu_generic.map_3d(path)
755
+ assert isinstance(result, np.ndarray)
756
+ assert result.dtype == np.float32
757
+ assert result.shape == (4, 5, 2)
758
+
759
+ def test_npy_values_match(self, map_3d_npy_path):
760
+ path, expected = map_3d_npy_path
761
+ result = cpu_generic.map_3d(path)
762
+ assert np.allclose(result, np.transpose(expected, (1, 2, 0)))
763
+
764
+
765
+ class TestMapWriters:
766
+ """Writer round-trip tests for ``map_2d`` / ``map_3d``."""
767
+
768
+ def test_gpu_write_map_2d_npy_roundtrip(self, tmp_path):
769
+ data = torch.rand(4, 5, dtype=torch.float32)
770
+ path = tmp_path / "m.npy"
771
+ gpu_generic.write_map_2d(str(path), data)
772
+ loaded = gpu_generic.map_2d(str(path))
773
+ assert torch.allclose(loaded, data)
774
+
775
+ def test_gpu_write_map_2d_accepts_1hw(self, tmp_path):
776
+ data = torch.rand(1, 4, 5, dtype=torch.float32)
777
+ path = tmp_path / "m.npy"
778
+ gpu_generic.write_map_2d(str(path), data)
779
+ loaded = gpu_generic.map_2d(str(path))
780
+ assert loaded.shape == (4, 5)
781
+ assert torch.allclose(loaded, data.squeeze(0))
782
+
783
+ def test_gpu_write_map_2d_npz_roundtrip(self, tmp_path):
784
+ data = torch.rand(4, 5, dtype=torch.float32)
785
+ path = tmp_path / "m.npz"
786
+ gpu_generic.write_map_2d(str(path), data)
787
+ loaded = gpu_generic.map_2d(str(path))
788
+ assert torch.allclose(loaded, data)
789
+
790
+ def test_cpu_write_map_2d_npy_roundtrip(self, tmp_path):
791
+ data = np.random.default_rng(0).random((4, 5)).astype(np.float32)
792
+ path = tmp_path / "m.npy"
793
+ cpu_generic.write_map_2d(str(path), data)
794
+ loaded = cpu_generic.map_2d(str(path))
795
+ assert np.allclose(loaded, data)
796
+
797
+ def test_gpu_write_map_3d_npy_roundtrip(self, tmp_path):
798
+ data = torch.rand(2, 4, 5, dtype=torch.float32)
799
+ path = tmp_path / "m.npy"
800
+ gpu_generic.write_map_3d(str(path), data)
801
+ loaded = gpu_generic.map_3d(str(path))
802
+ assert torch.allclose(loaded, data)
803
+
804
+ def test_cpu_write_map_3d_npy_roundtrip(self, tmp_path):
805
+ data = np.random.default_rng(0).random((4, 5, 2)).astype(np.float32)
806
+ path = tmp_path / "m.npy"
807
+ cpu_generic.write_map_3d(str(path), data)
808
+ loaded = cpu_generic.map_3d(str(path))
809
+ assert np.allclose(loaded, data)
810
+
811
+ def test_cpu_write_map_3d_npz_roundtrip(self, tmp_path):
812
+ data = np.random.default_rng(0).random((4, 5, 2)).astype(np.float32)
813
+ path = tmp_path / "m.npz"
814
+ cpu_generic.write_map_3d(str(path), data)
815
+ loaded = cpu_generic.map_3d(str(path))
816
+ assert np.allclose(loaded, data)
817
+
818
+ def test_gpu_to_cpu_map_3d_matches_transpose(self, tmp_path):
819
+ """A GPU-written map_3d reads back on CPU as the HWC transpose."""
820
+ data = torch.rand(2, 4, 5, dtype=torch.float32)
821
+ path = tmp_path / "m.npy"
822
+ gpu_generic.write_map_3d(str(path), data)
823
+ loaded = cpu_generic.map_3d(str(path))
824
+ assert np.allclose(loaded, data.permute(1, 2, 0).numpy())
File without changes
File without changes
File without changes