euler-loading 2.3.0__tar.gz → 2.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {euler_loading-2.3.0 → euler_loading-2.5.0}/PKG-INFO +1 -1
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/dataset.py +82 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/cpu/generic.py +72 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/generate/loaders.json +68 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/gpu/generic.py +60 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/package-lock.json +1 -1
- {euler_loading-2.3.0 → euler_loading-2.5.0}/pyproject.toml +1 -1
- euler_loading-2.5.0/tests/test_id_schema.py +116 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/test_loaders.py +186 -2
- {euler_loading-2.3.0 → euler_loading-2.5.0}/.github/workflows/workflow.yml +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/.gitignore +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/README.md +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/__init__.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/_dataset_contract.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/_ds_crawler_utils.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/_metadata.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/_resolution.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/_writing.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/indexing.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/__init__.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/_annotations.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/_writer_utils.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/contracts.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/cpu/__init__.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/cpu/generic_dense_depth.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/cpu/real_drive_sim.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/cpu/vkitti2.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/generate/__init__.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/generate/__main__.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/generic.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/gpu/__init__.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/gpu/generic_dense_depth.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/gpu/real_drive_sim.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/gpu/vkitti2.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/real_drive_sim.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/vkitti2.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/preprocessing.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/example.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/sample_rds.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/__init__.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/conftest.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/example_rds_calib.json +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/test_dataset.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/test_indexing.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/test_preprocessing.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/test_real_dataset.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/tests/test_writing.py +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/vkitti_cpu_example_output.json +0 -0
- {euler_loading-2.3.0 → euler_loading-2.5.0}/vkitti_gpu_example_output.json +0 -0
|
@@ -118,6 +118,36 @@ def _get_index_meta(index_output: Mapping[str, Any]) -> dict[str, Any] | None:
|
|
|
118
118
|
return None
|
|
119
119
|
|
|
120
120
|
|
|
121
|
+
def _extract_id_schema(index_output: Mapping[str, Any]) -> dict[str, Any]:
|
|
122
|
+
"""Pull id/hierarchy separators from a ds-crawler output payload.
|
|
123
|
+
|
|
124
|
+
Supports both modern (``indexing.id.join_char``) and legacy
|
|
125
|
+
(top-level ``id_regex_join_char``) layouts. Missing keys fall back to
|
|
126
|
+
ds-crawler defaults so the returned dict is always usable.
|
|
127
|
+
"""
|
|
128
|
+
indexing = index_output.get("indexing")
|
|
129
|
+
indexing_map = indexing if isinstance(indexing, Mapping) else {}
|
|
130
|
+
id_cfg = indexing_map.get("id")
|
|
131
|
+
id_map = id_cfg if isinstance(id_cfg, Mapping) else {}
|
|
132
|
+
hierarchy_cfg = indexing_map.get("hierarchy")
|
|
133
|
+
hierarchy_map = hierarchy_cfg if isinstance(hierarchy_cfg, Mapping) else {}
|
|
134
|
+
|
|
135
|
+
id_join_char = id_map.get("join_char") or index_output.get("id_regex_join_char") or "+"
|
|
136
|
+
hierarchy_separator = hierarchy_map.get("separator") or "-"
|
|
137
|
+
id_regex = id_map.get("regex") or index_output.get("id_regex")
|
|
138
|
+
hierarchy_regex = hierarchy_map.get("regex") or index_output.get("hierarchy_regex")
|
|
139
|
+
|
|
140
|
+
schema: dict[str, Any] = {
|
|
141
|
+
"hierarchy_separator": hierarchy_separator,
|
|
142
|
+
"id_join_char": id_join_char,
|
|
143
|
+
}
|
|
144
|
+
if id_regex:
|
|
145
|
+
schema["id_regex"] = id_regex
|
|
146
|
+
if hierarchy_regex:
|
|
147
|
+
schema["hierarchy_regex"] = hierarchy_regex
|
|
148
|
+
return schema
|
|
149
|
+
|
|
150
|
+
|
|
121
151
|
#TODO: might want to add slots=True in a 3.10+ only codebase
|
|
122
152
|
@dataclass(frozen=True)
|
|
123
153
|
class Modality:
|
|
@@ -488,6 +518,58 @@ class MultiModalDataset(_BaseDataset):
|
|
|
488
518
|
meta = _get_index_meta(index)
|
|
489
519
|
return meta or {}
|
|
490
520
|
|
|
521
|
+
def describe_id_schema(self) -> dict[str, Any]:
|
|
522
|
+
"""Return the id-construction schema used by this dataset.
|
|
523
|
+
|
|
524
|
+
Downstream consumers that want to split or match ``sample["full_id"]``
|
|
525
|
+
/ ``sample["id"]`` values back to ds-crawler fields need to know the
|
|
526
|
+
separators used to build them. This method exposes that schema so it
|
|
527
|
+
can be embedded in experiment manifests.
|
|
528
|
+
|
|
529
|
+
Schema keys:
|
|
530
|
+
- ``hierarchy_separator``: the character between the named
|
|
531
|
+
capture group *name* and its *value* within a single id part
|
|
532
|
+
(ds-crawler ``indexing.hierarchy.separator``, typically ``"-"``).
|
|
533
|
+
- ``id_join_char``: the character between id parts (ds-crawler
|
|
534
|
+
``indexing.id.join_char`` or legacy ``id_regex_join_char``,
|
|
535
|
+
default ``"+"``).
|
|
536
|
+
- ``full_id_separator``: the character between hierarchy levels
|
|
537
|
+
and the file id in ``sample["full_id"]`` (currently ``"/"``).
|
|
538
|
+
- ``id_regex`` / ``hierarchy_regex``: the raw regexes, when
|
|
539
|
+
recorded in the ds-crawler output.
|
|
540
|
+
- ``modalities``: per-modality overrides populated only when
|
|
541
|
+
modalities disagree on the above.
|
|
542
|
+
|
|
543
|
+
Example::
|
|
544
|
+
|
|
545
|
+
{
|
|
546
|
+
"hierarchy_separator": "-",
|
|
547
|
+
"id_join_char": "+",
|
|
548
|
+
"full_id_separator": "/",
|
|
549
|
+
"id_regex": "...",
|
|
550
|
+
}
|
|
551
|
+
"""
|
|
552
|
+
per_modality: dict[str, dict[str, Any]] = {}
|
|
553
|
+
for name, index in self._index_outputs.items():
|
|
554
|
+
per_modality[name] = _extract_id_schema(index)
|
|
555
|
+
|
|
556
|
+
if not per_modality:
|
|
557
|
+
return {}
|
|
558
|
+
|
|
559
|
+
base = dict(next(iter(per_modality.values())))
|
|
560
|
+
base["full_id_separator"] = "/"
|
|
561
|
+
|
|
562
|
+
overrides: dict[str, dict[str, Any]] = {}
|
|
563
|
+
for mod_name, mod_schema in per_modality.items():
|
|
564
|
+
mod_with_separator = dict(mod_schema)
|
|
565
|
+
mod_with_separator["full_id_separator"] = "/"
|
|
566
|
+
if mod_with_separator != base:
|
|
567
|
+
overrides[mod_name] = mod_with_separator
|
|
568
|
+
|
|
569
|
+
if overrides:
|
|
570
|
+
base["modalities"] = overrides
|
|
571
|
+
return base
|
|
572
|
+
|
|
491
573
|
def get_modality_index(self, modality_name: str) -> dict[str, Any]:
|
|
492
574
|
"""Return the cached ds-crawler index for a modality."""
|
|
493
575
|
if modality_name in self._modalities:
|
|
@@ -10,6 +10,8 @@ Supported file extensions:
|
|
|
10
10
|
|
|
11
11
|
Return types
|
|
12
12
|
------------
|
|
13
|
+
- **map_2d** -- ``np.ndarray`` of shape ``(H, W)`` float32.
|
|
14
|
+
- **map_3d** -- ``np.ndarray`` of shape ``(H, W, C)`` float32.
|
|
13
15
|
- **spherical_map** -- ``np.ndarray`` of shape ``(H, W, C)`` float32.
|
|
14
16
|
- **intrinsics** -- ``np.ndarray`` of shape ``(3, 3)`` float32.
|
|
15
17
|
- **sh_coeffs** -- ``np.ndarray`` of shape ``(N, 3)`` float32.
|
|
@@ -19,6 +21,8 @@ Usage::
|
|
|
19
21
|
from euler_loading.loaders.cpu import generic
|
|
20
22
|
from euler_loading import Modality
|
|
21
23
|
|
|
24
|
+
Modality("/data/my_dataset/scattering_coefficient", loader=generic.map_2d)
|
|
25
|
+
Modality("/data/my_dataset/atmospheric_light", loader=generic.map_3d)
|
|
22
26
|
Modality("/data/my_dataset/spherical_map", loader=generic.spherical_map)
|
|
23
27
|
Modality("/data/my_dataset/intrinsics", loader=generic.intrinsics)
|
|
24
28
|
Modality("/data/my_dataset/sh_coeffs", loader=generic.sh_coeffs)
|
|
@@ -36,6 +40,7 @@ from euler_loading.loaders._writer_utils import (
|
|
|
36
40
|
ensure_parent,
|
|
37
41
|
get_target_name,
|
|
38
42
|
mark_stream_supported,
|
|
43
|
+
to_hw,
|
|
39
44
|
to_numpy,
|
|
40
45
|
)
|
|
41
46
|
|
|
@@ -85,6 +90,40 @@ def _write_numpy(path: Union[str, BinaryIO], value: Any) -> None:
|
|
|
85
90
|
# ---------------------------------------------------------------------------
|
|
86
91
|
|
|
87
92
|
|
|
93
|
+
@modality_meta(
|
|
94
|
+
modality_type="map_2d",
|
|
95
|
+
dtype="float32",
|
|
96
|
+
shape="HW",
|
|
97
|
+
file_formats=[".npy", ".npz"],
|
|
98
|
+
)
|
|
99
|
+
def map_2d(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> np.ndarray:
|
|
100
|
+
"""Load an arbitrary 2D map as an ``(H, W)`` float32 array.
|
|
101
|
+
|
|
102
|
+
Suitable for any single-channel dense quantity (e.g. scattering
|
|
103
|
+
coefficient, attenuation, opacity) where no dataset-specific decoding
|
|
104
|
+
is required.
|
|
105
|
+
"""
|
|
106
|
+
return _load_numpy(path)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@modality_meta(
|
|
110
|
+
modality_type="map_3d",
|
|
111
|
+
dtype="float32",
|
|
112
|
+
shape="HWC",
|
|
113
|
+
file_formats=[".npy", ".npz"],
|
|
114
|
+
)
|
|
115
|
+
def map_3d(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> np.ndarray:
|
|
116
|
+
"""Load an arbitrary 3D map as an ``(H, W, C)`` float32 array.
|
|
117
|
+
|
|
118
|
+
The file is expected to be stored in ``(C, H, W)`` layout and is
|
|
119
|
+
transposed to ``(H, W, C)`` for CPU-oriented processing. Suitable
|
|
120
|
+
for any per-pixel vector quantity (e.g. atmospheric light, surface
|
|
121
|
+
normals, flow).
|
|
122
|
+
"""
|
|
123
|
+
arr = _load_numpy(path)
|
|
124
|
+
return np.transpose(arr, (1, 2, 0))
|
|
125
|
+
|
|
126
|
+
|
|
88
127
|
@modality_meta(
|
|
89
128
|
modality_type="spherical_map",
|
|
90
129
|
dtype="float32",
|
|
@@ -145,6 +184,39 @@ def sh_coeffs(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) ->
|
|
|
145
184
|
# ---------------------------------------------------------------------------
|
|
146
185
|
|
|
147
186
|
|
|
187
|
+
@mark_stream_supported
|
|
188
|
+
def write_map_2d(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
|
|
189
|
+
"""Write a 2D map to NumPy formats based on extension.
|
|
190
|
+
|
|
191
|
+
Accepts ``(H, W)``, ``(1, H, W)``, or ``(H, W, 1)`` input and stores
|
|
192
|
+
the array in ``(H, W)`` layout.
|
|
193
|
+
"""
|
|
194
|
+
arr = to_hw(value, name="map_2d")
|
|
195
|
+
_write_numpy(path, arr)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
@mark_stream_supported
|
|
199
|
+
def write_map_3d(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
|
|
200
|
+
"""Write a 3D map to NumPy formats based on extension.
|
|
201
|
+
|
|
202
|
+
Input is expected in ``(H, W, C)`` layout and is transposed to
|
|
203
|
+
``(C, H, W)`` for storage.
|
|
204
|
+
"""
|
|
205
|
+
ensure_parent(path)
|
|
206
|
+
ext = os.path.splitext(_get_name(path))[1].lower()
|
|
207
|
+
arr = to_numpy(value).astype(np.float32)
|
|
208
|
+
arr = np.transpose(arr, (2, 0, 1))
|
|
209
|
+
|
|
210
|
+
if ext == _NPY_EXTENSION:
|
|
211
|
+
np.save(path, arr)
|
|
212
|
+
return
|
|
213
|
+
if ext == _NPZ_EXTENSION:
|
|
214
|
+
np.savez_compressed(path, data=arr)
|
|
215
|
+
return
|
|
216
|
+
|
|
217
|
+
raise ValueError(f"Unsupported map_3d output extension: {ext}")
|
|
218
|
+
|
|
219
|
+
|
|
148
220
|
@mark_stream_supported
|
|
149
221
|
def write_spherical_map(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
|
|
150
222
|
"""Write spherical-map data to NumPy formats based on extension.
|
|
@@ -3,6 +3,74 @@
|
|
|
3
3
|
{
|
|
4
4
|
"name": "generic",
|
|
5
5
|
"modalities": [
|
|
6
|
+
{
|
|
7
|
+
"type": "intrinsics",
|
|
8
|
+
"function": "intrinsics",
|
|
9
|
+
"hierarchical": false,
|
|
10
|
+
"dtype": "float32",
|
|
11
|
+
"access": "key",
|
|
12
|
+
"file_formats": [
|
|
13
|
+
".npy",
|
|
14
|
+
".npz"
|
|
15
|
+
],
|
|
16
|
+
"cpu": {
|
|
17
|
+
"shape": "3x3"
|
|
18
|
+
},
|
|
19
|
+
"gpu": {
|
|
20
|
+
"shape": "3x3"
|
|
21
|
+
}
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"type": "map_2d",
|
|
25
|
+
"function": "map_2d",
|
|
26
|
+
"hierarchical": false,
|
|
27
|
+
"dtype": "float32",
|
|
28
|
+
"access": "key",
|
|
29
|
+
"file_formats": [
|
|
30
|
+
".npy",
|
|
31
|
+
".npz"
|
|
32
|
+
],
|
|
33
|
+
"cpu": {
|
|
34
|
+
"shape": "HW"
|
|
35
|
+
},
|
|
36
|
+
"gpu": {
|
|
37
|
+
"shape": "HW"
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"type": "map_3d",
|
|
42
|
+
"function": "map_3d",
|
|
43
|
+
"hierarchical": false,
|
|
44
|
+
"dtype": "float32",
|
|
45
|
+
"access": "key",
|
|
46
|
+
"file_formats": [
|
|
47
|
+
".npy",
|
|
48
|
+
".npz"
|
|
49
|
+
],
|
|
50
|
+
"cpu": {
|
|
51
|
+
"shape": "HWC"
|
|
52
|
+
},
|
|
53
|
+
"gpu": {
|
|
54
|
+
"shape": "CHW"
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"type": "sh_coeffs",
|
|
59
|
+
"function": "sh_coeffs",
|
|
60
|
+
"hierarchical": false,
|
|
61
|
+
"dtype": "float32",
|
|
62
|
+
"access": "key",
|
|
63
|
+
"file_formats": [
|
|
64
|
+
".npy",
|
|
65
|
+
".npz"
|
|
66
|
+
],
|
|
67
|
+
"cpu": {
|
|
68
|
+
"shape": "NC"
|
|
69
|
+
},
|
|
70
|
+
"gpu": {
|
|
71
|
+
"shape": "NC"
|
|
72
|
+
}
|
|
73
|
+
},
|
|
6
74
|
{
|
|
7
75
|
"type": "spherical_map",
|
|
8
76
|
"function": "spherical_map",
|
|
@@ -10,6 +10,8 @@ Supported file extensions:
|
|
|
10
10
|
|
|
11
11
|
Return types
|
|
12
12
|
------------
|
|
13
|
+
- **map_2d** -- ``torch.FloatTensor`` of shape ``(H, W)``.
|
|
14
|
+
- **map_3d** -- ``torch.FloatTensor`` of shape ``(C, H, W)``.
|
|
13
15
|
- **spherical_map** -- ``torch.FloatTensor`` of shape ``(C, H, W)``.
|
|
14
16
|
- **intrinsics** -- ``torch.FloatTensor`` of shape ``(3, 3)``.
|
|
15
17
|
- **sh_coeffs** -- ``torch.FloatTensor`` of shape ``(N, 3)``.
|
|
@@ -19,6 +21,8 @@ Usage::
|
|
|
19
21
|
from euler_loading.loaders.gpu import generic
|
|
20
22
|
from euler_loading import Modality
|
|
21
23
|
|
|
24
|
+
Modality("/data/my_dataset/scattering_coefficient", loader=generic.map_2d)
|
|
25
|
+
Modality("/data/my_dataset/atmospheric_light", loader=generic.map_3d)
|
|
22
26
|
Modality("/data/my_dataset/spherical_map", loader=generic.spherical_map)
|
|
23
27
|
Modality("/data/my_dataset/intrinsics", loader=generic.intrinsics)
|
|
24
28
|
Modality("/data/my_dataset/sh_coeffs", loader=generic.sh_coeffs)
|
|
@@ -37,6 +41,7 @@ from euler_loading.loaders._writer_utils import (
|
|
|
37
41
|
ensure_parent,
|
|
38
42
|
get_target_name,
|
|
39
43
|
mark_stream_supported,
|
|
44
|
+
to_hw,
|
|
40
45
|
to_numpy,
|
|
41
46
|
)
|
|
42
47
|
|
|
@@ -86,6 +91,40 @@ def _write_numpy(path: Union[str, BinaryIO], value: Any) -> None:
|
|
|
86
91
|
# ---------------------------------------------------------------------------
|
|
87
92
|
|
|
88
93
|
|
|
94
|
+
@modality_meta(
|
|
95
|
+
modality_type="map_2d",
|
|
96
|
+
dtype="float32",
|
|
97
|
+
shape="HW",
|
|
98
|
+
file_formats=[".npy", ".npz"],
|
|
99
|
+
)
|
|
100
|
+
def map_2d(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> torch.Tensor:
|
|
101
|
+
"""Load an arbitrary 2D map as an ``(H, W)`` float32 tensor.
|
|
102
|
+
|
|
103
|
+
Suitable for any single-channel dense quantity (e.g. scattering
|
|
104
|
+
coefficient, attenuation, opacity) where no dataset-specific decoding
|
|
105
|
+
is required.
|
|
106
|
+
"""
|
|
107
|
+
arr = _load_numpy(path)
|
|
108
|
+
return torch.from_numpy(arr).contiguous()
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@modality_meta(
|
|
112
|
+
modality_type="map_3d",
|
|
113
|
+
dtype="float32",
|
|
114
|
+
shape="CHW",
|
|
115
|
+
file_formats=[".npy", ".npz"],
|
|
116
|
+
)
|
|
117
|
+
def map_3d(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> torch.Tensor:
|
|
118
|
+
"""Load an arbitrary 3D map as a ``(C, H, W)`` float32 tensor.
|
|
119
|
+
|
|
120
|
+
The file is expected to already be stored in ``(C, H, W)`` layout,
|
|
121
|
+
matching the torch convention. Suitable for any per-pixel vector
|
|
122
|
+
quantity (e.g. atmospheric light, surface normals, flow).
|
|
123
|
+
"""
|
|
124
|
+
arr = _load_numpy(path)
|
|
125
|
+
return torch.from_numpy(arr).contiguous()
|
|
126
|
+
|
|
127
|
+
|
|
89
128
|
@modality_meta(
|
|
90
129
|
modality_type="spherical_map",
|
|
91
130
|
dtype="float32",
|
|
@@ -147,6 +186,27 @@ def sh_coeffs(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) ->
|
|
|
147
186
|
# ---------------------------------------------------------------------------
|
|
148
187
|
|
|
149
188
|
|
|
189
|
+
@mark_stream_supported
|
|
190
|
+
def write_map_2d(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
|
|
191
|
+
"""Write a 2D map to NumPy formats based on extension.
|
|
192
|
+
|
|
193
|
+
Accepts ``(H, W)``, ``(1, H, W)``, or ``(H, W, 1)`` input and stores
|
|
194
|
+
the array in ``(H, W)`` layout.
|
|
195
|
+
"""
|
|
196
|
+
arr = to_hw(value, name="map_2d")
|
|
197
|
+
_write_numpy(path, arr)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
@mark_stream_supported
|
|
201
|
+
def write_map_3d(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
|
|
202
|
+
"""Write a 3D map to NumPy formats based on extension.
|
|
203
|
+
|
|
204
|
+
Input is expected in ``(C, H, W)`` layout (torch convention) and is
|
|
205
|
+
stored as-is.
|
|
206
|
+
"""
|
|
207
|
+
_write_numpy(path, value)
|
|
208
|
+
|
|
209
|
+
|
|
150
210
|
@mark_stream_supported
|
|
151
211
|
def write_spherical_map(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
|
|
152
212
|
"""Write spherical-map data to NumPy formats based on extension."""
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Tests for MultiModalDataset.describe_id_schema."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
from unittest.mock import patch
|
|
7
|
+
|
|
8
|
+
from euler_loading import Modality, MultiModalDataset
|
|
9
|
+
|
|
10
|
+
from .conftest import dummy_loader
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _flat_index(
|
|
14
|
+
file_ids: list[str],
|
|
15
|
+
*,
|
|
16
|
+
indexing: dict[str, Any] | None = None,
|
|
17
|
+
legacy_join_char: str | None = None,
|
|
18
|
+
) -> dict[str, Any]:
|
|
19
|
+
index: dict[str, Any] = {
|
|
20
|
+
"name": "test-ds",
|
|
21
|
+
"type": "depth",
|
|
22
|
+
"euler_train": {"used_as": "input", "modality_type": "depth"},
|
|
23
|
+
"dataset": {
|
|
24
|
+
"files": [
|
|
25
|
+
{
|
|
26
|
+
"id": fid,
|
|
27
|
+
"path": f"scene/{fid}.png",
|
|
28
|
+
"path_properties": {},
|
|
29
|
+
"basename_properties": {},
|
|
30
|
+
}
|
|
31
|
+
for fid in file_ids
|
|
32
|
+
]
|
|
33
|
+
},
|
|
34
|
+
}
|
|
35
|
+
if indexing is not None:
|
|
36
|
+
index["indexing"] = indexing
|
|
37
|
+
if legacy_join_char is not None:
|
|
38
|
+
index["id_regex_join_char"] = legacy_join_char
|
|
39
|
+
return index
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class TestDescribeIdSchema:
|
|
43
|
+
def test_modern_indexing_block(self):
|
|
44
|
+
index = _flat_index(
|
|
45
|
+
["f001"],
|
|
46
|
+
indexing={
|
|
47
|
+
"id": {"regex": "^(.+)$", "join_char": "+"},
|
|
48
|
+
"hierarchy": {"separator": "-"},
|
|
49
|
+
},
|
|
50
|
+
)
|
|
51
|
+
with patch(
|
|
52
|
+
"euler_loading.dataset.index_dataset_from_path",
|
|
53
|
+
return_value=index,
|
|
54
|
+
):
|
|
55
|
+
ds = MultiModalDataset(
|
|
56
|
+
modalities={"depth": Modality("/data/depth", loader=dummy_loader)}
|
|
57
|
+
)
|
|
58
|
+
schema = ds.describe_id_schema()
|
|
59
|
+
assert schema["hierarchy_separator"] == "-"
|
|
60
|
+
assert schema["id_join_char"] == "+"
|
|
61
|
+
assert schema["full_id_separator"] == "/"
|
|
62
|
+
assert schema["id_regex"] == "^(.+)$"
|
|
63
|
+
|
|
64
|
+
def test_legacy_top_level_join_char(self):
|
|
65
|
+
index = _flat_index(["f001"], legacy_join_char="|")
|
|
66
|
+
with patch(
|
|
67
|
+
"euler_loading.dataset.index_dataset_from_path",
|
|
68
|
+
return_value=index,
|
|
69
|
+
):
|
|
70
|
+
ds = MultiModalDataset(
|
|
71
|
+
modalities={"depth": Modality("/data/depth", loader=dummy_loader)}
|
|
72
|
+
)
|
|
73
|
+
schema = ds.describe_id_schema()
|
|
74
|
+
assert schema["id_join_char"] == "|"
|
|
75
|
+
assert schema["hierarchy_separator"] == "-"
|
|
76
|
+
|
|
77
|
+
def test_missing_indexing_falls_back_to_defaults(self):
|
|
78
|
+
index = _flat_index(["f001"])
|
|
79
|
+
with patch(
|
|
80
|
+
"euler_loading.dataset.index_dataset_from_path",
|
|
81
|
+
return_value=index,
|
|
82
|
+
):
|
|
83
|
+
ds = MultiModalDataset(
|
|
84
|
+
modalities={"depth": Modality("/data/depth", loader=dummy_loader)}
|
|
85
|
+
)
|
|
86
|
+
schema = ds.describe_id_schema()
|
|
87
|
+
assert schema["hierarchy_separator"] == "-"
|
|
88
|
+
assert schema["id_join_char"] == "+"
|
|
89
|
+
assert schema["full_id_separator"] == "/"
|
|
90
|
+
|
|
91
|
+
def test_divergent_modalities_record_overrides(self):
|
|
92
|
+
rgb_index = _flat_index(
|
|
93
|
+
["f001"],
|
|
94
|
+
indexing={"id": {"join_char": "+"}, "hierarchy": {"separator": "-"}},
|
|
95
|
+
)
|
|
96
|
+
depth_index = _flat_index(
|
|
97
|
+
["f001"],
|
|
98
|
+
indexing={"id": {"join_char": "|"}, "hierarchy": {"separator": "-"}},
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
def mock_index(path: str, **_: Any) -> dict[str, Any]:
|
|
102
|
+
return rgb_index if "rgb" in path else depth_index
|
|
103
|
+
|
|
104
|
+
with patch(
|
|
105
|
+
"euler_loading.dataset.index_dataset_from_path",
|
|
106
|
+
side_effect=mock_index,
|
|
107
|
+
):
|
|
108
|
+
ds = MultiModalDataset(
|
|
109
|
+
modalities={
|
|
110
|
+
"rgb": Modality("/data/rgb", loader=dummy_loader),
|
|
111
|
+
"depth": Modality("/data/depth", loader=dummy_loader),
|
|
112
|
+
}
|
|
113
|
+
)
|
|
114
|
+
schema = ds.describe_id_schema()
|
|
115
|
+
assert "modalities" in schema
|
|
116
|
+
assert schema["modalities"]["depth"]["id_join_char"] == "|"
|
|
@@ -485,8 +485,8 @@ class TestRDSWriters:
|
|
|
485
485
|
# Generic spherical_map loader tests
|
|
486
486
|
# ---------------------------------------------------------------------------
|
|
487
487
|
|
|
488
|
-
GENERIC_LOADER_NAMES = ["spherical_map"]
|
|
489
|
-
GENERIC_WRITER_NAMES = ["write_spherical_map"]
|
|
488
|
+
GENERIC_LOADER_NAMES = ["map_2d", "map_3d", "spherical_map"]
|
|
489
|
+
GENERIC_WRITER_NAMES = ["write_map_2d", "write_map_3d", "write_spherical_map"]
|
|
490
490
|
|
|
491
491
|
|
|
492
492
|
@pytest.fixture()
|
|
@@ -638,3 +638,187 @@ class TestGenericBackwardCompat:
|
|
|
638
638
|
top = generic_top.spherical_map(path)
|
|
639
639
|
gpu = gpu_generic.spherical_map(path)
|
|
640
640
|
assert torch.equal(top, gpu)
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
# ---------------------------------------------------------------------------
|
|
644
|
+
# Generic map_2d / map_3d loader tests
|
|
645
|
+
# ---------------------------------------------------------------------------
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
@pytest.fixture()
|
|
649
|
+
def map_2d_npy_path(tmp_path):
|
|
650
|
+
"""Write a small (4, 5) float32 .npy file."""
|
|
651
|
+
arr = np.random.default_rng(0).random((4, 5), dtype=np.float32)
|
|
652
|
+
p = tmp_path / "map_2d.npy"
|
|
653
|
+
np.save(str(p), arr)
|
|
654
|
+
return str(p), arr
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
@pytest.fixture()
|
|
658
|
+
def map_2d_npz_path(tmp_path):
|
|
659
|
+
"""Write a small (4, 5) float32 .npz file."""
|
|
660
|
+
arr = np.random.default_rng(1).random((4, 5), dtype=np.float32)
|
|
661
|
+
p = tmp_path / "map_2d.npz"
|
|
662
|
+
np.savez_compressed(str(p), data=arr)
|
|
663
|
+
return str(p), arr
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
@pytest.fixture()
|
|
667
|
+
def map_3d_npy_path(tmp_path):
|
|
668
|
+
"""Write a small (2, 4, 5) float32 .npy file in CHW layout."""
|
|
669
|
+
arr = np.random.default_rng(2).random((2, 4, 5), dtype=np.float32)
|
|
670
|
+
p = tmp_path / "map_3d.npy"
|
|
671
|
+
np.save(str(p), arr)
|
|
672
|
+
return str(p), arr
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
@pytest.fixture()
|
|
676
|
+
def map_3d_npz_path(tmp_path):
|
|
677
|
+
"""Write a small (2, 4, 5) float32 .npz file in CHW layout."""
|
|
678
|
+
arr = np.random.default_rng(3).random((2, 4, 5), dtype=np.float32)
|
|
679
|
+
p = tmp_path / "map_3d.npz"
|
|
680
|
+
np.savez_compressed(str(p), data=arr)
|
|
681
|
+
return str(p), arr
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
class TestGPUMap2DLoader:
|
|
685
|
+
"""GPU ``map_2d`` loads a 2D map as an ``(H, W)`` float32 tensor."""
|
|
686
|
+
|
|
687
|
+
def test_npy_dtype_and_shape(self, map_2d_npy_path):
|
|
688
|
+
path, _ = map_2d_npy_path
|
|
689
|
+
result = gpu_generic.map_2d(path)
|
|
690
|
+
assert isinstance(result, torch.Tensor)
|
|
691
|
+
assert result.dtype == torch.float32
|
|
692
|
+
assert result.shape == (4, 5)
|
|
693
|
+
|
|
694
|
+
def test_npy_values_match(self, map_2d_npy_path):
|
|
695
|
+
path, expected = map_2d_npy_path
|
|
696
|
+
result = gpu_generic.map_2d(path)
|
|
697
|
+
assert torch.allclose(result, torch.from_numpy(expected))
|
|
698
|
+
|
|
699
|
+
def test_npz_dtype_and_shape(self, map_2d_npz_path):
|
|
700
|
+
path, _ = map_2d_npz_path
|
|
701
|
+
result = gpu_generic.map_2d(path)
|
|
702
|
+
assert result.dtype == torch.float32
|
|
703
|
+
assert result.shape == (4, 5)
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
class TestCPUMap2DLoader:
|
|
707
|
+
"""CPU ``map_2d`` loads a 2D map as an ``(H, W)`` float32 array."""
|
|
708
|
+
|
|
709
|
+
def test_npy_dtype_and_shape(self, map_2d_npy_path):
|
|
710
|
+
path, _ = map_2d_npy_path
|
|
711
|
+
result = cpu_generic.map_2d(path)
|
|
712
|
+
assert isinstance(result, np.ndarray)
|
|
713
|
+
assert result.dtype == np.float32
|
|
714
|
+
assert result.shape == (4, 5)
|
|
715
|
+
|
|
716
|
+
def test_npy_values_match(self, map_2d_npy_path):
|
|
717
|
+
path, expected = map_2d_npy_path
|
|
718
|
+
result = cpu_generic.map_2d(path)
|
|
719
|
+
assert np.allclose(result, expected)
|
|
720
|
+
|
|
721
|
+
def test_npz_dtype_and_shape(self, map_2d_npz_path):
|
|
722
|
+
path, _ = map_2d_npz_path
|
|
723
|
+
result = cpu_generic.map_2d(path)
|
|
724
|
+
assert result.dtype == np.float32
|
|
725
|
+
assert result.shape == (4, 5)
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
class TestGPUMap3DLoader:
|
|
729
|
+
"""GPU ``map_3d`` loads a 3D map as a ``(C, H, W)`` float32 tensor."""
|
|
730
|
+
|
|
731
|
+
def test_npy_dtype_and_shape(self, map_3d_npy_path):
|
|
732
|
+
path, _ = map_3d_npy_path
|
|
733
|
+
result = gpu_generic.map_3d(path)
|
|
734
|
+
assert isinstance(result, torch.Tensor)
|
|
735
|
+
assert result.dtype == torch.float32
|
|
736
|
+
assert result.shape == (2, 4, 5)
|
|
737
|
+
|
|
738
|
+
def test_npy_values_match(self, map_3d_npy_path):
|
|
739
|
+
path, expected = map_3d_npy_path
|
|
740
|
+
result = gpu_generic.map_3d(path)
|
|
741
|
+
assert torch.allclose(result, torch.from_numpy(expected))
|
|
742
|
+
|
|
743
|
+
def test_npz_dtype_and_shape(self, map_3d_npz_path):
|
|
744
|
+
path, _ = map_3d_npz_path
|
|
745
|
+
result = gpu_generic.map_3d(path)
|
|
746
|
+
assert result.shape == (2, 4, 5)
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
class TestCPUMap3DLoader:
|
|
750
|
+
"""CPU ``map_3d`` transposes CHW-on-disk to ``(H, W, C)``."""
|
|
751
|
+
|
|
752
|
+
def test_npy_dtype_and_shape(self, map_3d_npy_path):
|
|
753
|
+
path, _ = map_3d_npy_path
|
|
754
|
+
result = cpu_generic.map_3d(path)
|
|
755
|
+
assert isinstance(result, np.ndarray)
|
|
756
|
+
assert result.dtype == np.float32
|
|
757
|
+
assert result.shape == (4, 5, 2)
|
|
758
|
+
|
|
759
|
+
def test_npy_values_match(self, map_3d_npy_path):
|
|
760
|
+
path, expected = map_3d_npy_path
|
|
761
|
+
result = cpu_generic.map_3d(path)
|
|
762
|
+
assert np.allclose(result, np.transpose(expected, (1, 2, 0)))
|
|
763
|
+
|
|
764
|
+
|
|
765
|
+
class TestMapWriters:
|
|
766
|
+
"""Writer round-trip tests for ``map_2d`` / ``map_3d``."""
|
|
767
|
+
|
|
768
|
+
def test_gpu_write_map_2d_npy_roundtrip(self, tmp_path):
|
|
769
|
+
data = torch.rand(4, 5, dtype=torch.float32)
|
|
770
|
+
path = tmp_path / "m.npy"
|
|
771
|
+
gpu_generic.write_map_2d(str(path), data)
|
|
772
|
+
loaded = gpu_generic.map_2d(str(path))
|
|
773
|
+
assert torch.allclose(loaded, data)
|
|
774
|
+
|
|
775
|
+
def test_gpu_write_map_2d_accepts_1hw(self, tmp_path):
|
|
776
|
+
data = torch.rand(1, 4, 5, dtype=torch.float32)
|
|
777
|
+
path = tmp_path / "m.npy"
|
|
778
|
+
gpu_generic.write_map_2d(str(path), data)
|
|
779
|
+
loaded = gpu_generic.map_2d(str(path))
|
|
780
|
+
assert loaded.shape == (4, 5)
|
|
781
|
+
assert torch.allclose(loaded, data.squeeze(0))
|
|
782
|
+
|
|
783
|
+
def test_gpu_write_map_2d_npz_roundtrip(self, tmp_path):
|
|
784
|
+
data = torch.rand(4, 5, dtype=torch.float32)
|
|
785
|
+
path = tmp_path / "m.npz"
|
|
786
|
+
gpu_generic.write_map_2d(str(path), data)
|
|
787
|
+
loaded = gpu_generic.map_2d(str(path))
|
|
788
|
+
assert torch.allclose(loaded, data)
|
|
789
|
+
|
|
790
|
+
def test_cpu_write_map_2d_npy_roundtrip(self, tmp_path):
|
|
791
|
+
data = np.random.default_rng(0).random((4, 5)).astype(np.float32)
|
|
792
|
+
path = tmp_path / "m.npy"
|
|
793
|
+
cpu_generic.write_map_2d(str(path), data)
|
|
794
|
+
loaded = cpu_generic.map_2d(str(path))
|
|
795
|
+
assert np.allclose(loaded, data)
|
|
796
|
+
|
|
797
|
+
def test_gpu_write_map_3d_npy_roundtrip(self, tmp_path):
|
|
798
|
+
data = torch.rand(2, 4, 5, dtype=torch.float32)
|
|
799
|
+
path = tmp_path / "m.npy"
|
|
800
|
+
gpu_generic.write_map_3d(str(path), data)
|
|
801
|
+
loaded = gpu_generic.map_3d(str(path))
|
|
802
|
+
assert torch.allclose(loaded, data)
|
|
803
|
+
|
|
804
|
+
def test_cpu_write_map_3d_npy_roundtrip(self, tmp_path):
|
|
805
|
+
data = np.random.default_rng(0).random((4, 5, 2)).astype(np.float32)
|
|
806
|
+
path = tmp_path / "m.npy"
|
|
807
|
+
cpu_generic.write_map_3d(str(path), data)
|
|
808
|
+
loaded = cpu_generic.map_3d(str(path))
|
|
809
|
+
assert np.allclose(loaded, data)
|
|
810
|
+
|
|
811
|
+
def test_cpu_write_map_3d_npz_roundtrip(self, tmp_path):
|
|
812
|
+
data = np.random.default_rng(0).random((4, 5, 2)).astype(np.float32)
|
|
813
|
+
path = tmp_path / "m.npz"
|
|
814
|
+
cpu_generic.write_map_3d(str(path), data)
|
|
815
|
+
loaded = cpu_generic.map_3d(str(path))
|
|
816
|
+
assert np.allclose(loaded, data)
|
|
817
|
+
|
|
818
|
+
def test_gpu_to_cpu_map_3d_matches_transpose(self, tmp_path):
|
|
819
|
+
"""A GPU-written map_3d reads back on CPU as the HWC transpose."""
|
|
820
|
+
data = torch.rand(2, 4, 5, dtype=torch.float32)
|
|
821
|
+
path = tmp_path / "m.npy"
|
|
822
|
+
gpu_generic.write_map_3d(str(path), data)
|
|
823
|
+
loaded = cpu_generic.map_3d(str(path))
|
|
824
|
+
assert np.allclose(loaded, data.permute(1, 2, 0).numpy())
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/cpu/generic_dense_depth.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{euler_loading-2.3.0 → euler_loading-2.5.0}/euler_loading/loaders/gpu/generic_dense_depth.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|