euler-loading 2.10.0__tar.gz → 2.12.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {euler_loading-2.10.0 → euler_loading-2.12.0}/PKG-INFO +1 -1
- {euler_loading-2.10.0 → euler_loading-2.12.0}/README.md +43 -3
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/_ds_crawler_utils.py +159 -7
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/dataset.py +98 -10
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/cpu/muses.py +79 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/cpu/real_drive_sim.py +134 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/generate/loaders.json +26 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/gpu/muses.py +80 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/pyproject.toml +1 -1
- {euler_loading-2.10.0 → euler_loading-2.12.0}/tests/test_dataset.py +190 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/tests/test_loaders.py +103 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/.github/workflows/workflow.yml +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/.gitignore +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/docs/loader-attributes.md +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/__init__.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/_dataset_contract.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/_metadata.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/_resolution.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/_writing.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/indexing.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/__init__.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/_annotations.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/_writer_utils.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/contracts.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/cpu/__init__.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/cpu/generic.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/cpu/generic_dense_depth.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/cpu/vkitti2.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/generate/__init__.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/generate/__main__.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/generic.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/gpu/__init__.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/gpu/generic.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/gpu/generic_dense_depth.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/gpu/real_drive_sim.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/gpu/vkitti2.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/muses.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/real_drive_sim.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/loaders/vkitti2.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/euler_loading/preprocessing.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/example.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/package-lock.json +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/sample_rds.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/tests/__init__.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/tests/conftest.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/tests/example_rds_calib.json +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/tests/test_id_schema.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/tests/test_indexing.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/tests/test_preprocessing.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/tests/test_real_dataset.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/tests/test_writing.py +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/vkitti_cpu_example_output.json +0 -0
- {euler_loading-2.10.0 → euler_loading-2.12.0}/vkitti_gpu_example_output.json +0 -0
|
@@ -55,7 +55,7 @@ Frozen dataclass describing one data modality.
|
|
|
55
55
|
|
|
56
56
|
| Parameter | Type | Description |
|
|
57
57
|
|-----------|------|-------------|
|
|
58
|
-
| `path` | `str` | Absolute path to the modality root directory or `.zip` archive. Must contain ds-crawler metadata, either at `.ds_crawler/` or under the configured `metadata_scope`. |
|
|
58
|
+
| `path` | `str` | Absolute path to the modality root directory or `.zip` archive. Must contain ds-crawler metadata, either at `.ds_crawler/` or under the configured `metadata_scope`. Inline selectors are accepted as `/data/ds.zip:train`, `/data/ds.zip#scope=rgb`, or `/data/ds.zip:train#scope=rgb`. |
|
|
59
59
|
| `origin_path` | `str \| None` | Original path before copying/symlinking (e.g. for SLURM staging). Not used by euler-loading itself — useful for experiment logging to retain references to the original dataset location. |
|
|
60
60
|
| `loader` | `Callable[..., Any] \| None` | Receives the file path (or `BinaryIO` buffer for zip-backed modalities) and an optional `meta` dict. Returns loaded data. When `None`, the loader is resolved automatically from the ds-crawler index (see [Automatic loader resolution](#automatic-loader-resolution)). |
|
|
61
61
|
| `writer` | `Callable[..., Any] \| None` | Receives `(path, value, meta)` and writes modality data to disk. When `None`, euler-loading tries to resolve a built-in writer from ds-crawler metadata (`write_<function>` or `write_<suffix>` for `read_<suffix>`). |
|
|
@@ -65,7 +65,7 @@ Frozen dataclass describing one data modality.
|
|
|
65
65
|
| `hierarchy_scope` | `str \| None` | Optional scope label for hierarchical modalities (e.g. `scene_camera`). |
|
|
66
66
|
| `applies_to` | `list[str] \| None` | Optional list of regular modality names a hierarchical modality applies to. |
|
|
67
67
|
| `split` | `str \| None` | Optional inline split name. Loads `.ds_crawler/split_<name>.json` from the modality root (directory or zip) and overlays it on the normal ds-crawler metadata. |
|
|
68
|
-
| `metadata_scope` | `str \| None` | Optional namespace below `.ds_crawler`, e.g. `.ds_crawler/camera_extrinsics/index.json`. Use this when multiple logical modalities share one physical directory or zip. If the
|
|
68
|
+
| `metadata_scope` | `str \| None` | Optional namespace below `.ds_crawler`, e.g. `.ds_crawler/camera_extrinsics/index.json`. Use this when multiple logical modalities share one physical directory or zip. It can also be supplied inline as `#scope=<metadata_scope>`. If omitted, euler-loading infers it only when the choice is deterministic. If configured artifacts are absent, loading falls back to the legacy root-level `.ds_crawler` layout. |
|
|
69
69
|
| `cache` | `bool \| None` | Opt-in/out for in-memory caching of decoded values (only meaningful for hierarchical modalities; regular modalities are never cached). `True` keeps every distinct loaded file in a process-lifetime dict. `False` re-reads on every access. `None` (default): hierarchical → `True` because small shared calibration files benefit. |
|
|
70
70
|
| `collapse_single` | `bool` | For hierarchical modalities only. When `True`, a sample that matches exactly one hierarchical file returns that loaded value directly instead of `{file_id: value}`. This is useful for shared GT tensors used by multiple augmentations of the same source sample. |
|
|
71
71
|
| `metadata` | `dict[str, Any]` | Optional arbitrary metadata. Keys under `metadata["euler_loading"]` are treated as euler-loading defaults. |
|
|
@@ -128,6 +128,12 @@ dataset = MultiModalDataset(
|
|
|
128
128
|
)
|
|
129
129
|
```
|
|
130
130
|
|
|
131
|
+
Path-only callers can use the existing colon form directly:
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
Modality("/data/rgb:train")
|
|
135
|
+
```
|
|
136
|
+
|
|
131
137
|
This works for both directory-backed and zip-backed modalities. The split file only replaces the `dataset` payload; top-level metadata such as dataset type and euler-loading loader hints still come from the canonical ds-crawler index.
|
|
132
138
|
|
|
133
139
|
### Scoped ds-crawler Metadata
|
|
@@ -165,6 +171,23 @@ dataset = MultiModalDataset(
|
|
|
165
171
|
)
|
|
166
172
|
```
|
|
167
173
|
|
|
174
|
+
Path-only callers can select the same scope inline. The scope selector comes
|
|
175
|
+
after the optional split selector:
|
|
176
|
+
|
|
177
|
+
```python
|
|
178
|
+
Modality("/data/muses.zip#scope=rgb")
|
|
179
|
+
Modality("/data/muses.zip:train#scope=rgb")
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
If `metadata_scope` is omitted, euler-loading resolves it automatically only
|
|
183
|
+
when doing so is unambiguous:
|
|
184
|
+
|
|
185
|
+
1. Explicit `metadata_scope` or `#scope=...` wins.
|
|
186
|
+
2. If root-level `.ds_crawler` metadata exists, the legacy root metadata is used.
|
|
187
|
+
3. If no root metadata exists and exactly one scope exists, that scope is used.
|
|
188
|
+
4. If multiple scopes exist, euler-loading matches the modality dict key, `modality_type`, loader `_modality_meta["type"]`, or scoped `dataset-head.json` `modality.key`.
|
|
189
|
+
5. If the result is still ambiguous, construction raises and lists the available scopes.
|
|
190
|
+
|
|
168
191
|
This is an additive layout. Existing roots with `.ds_crawler/index.json` continue to load unchanged, and a scoped modality falls back to that legacy location when the scoped artifacts are not present.
|
|
169
192
|
|
|
170
193
|
### `MultiModalDataset.modality_paths()`
|
|
@@ -193,6 +216,17 @@ PyTorch `Dataset`. On construction it:
|
|
|
193
216
|
| `hierarchical_modalities` | `dict[str, Modality] \| None` | Optional modalities whose files live at intermediate hierarchy levels (e.g. per-scene intrinsics). These do **not** participate in ID intersection. Each sample will contain a dict `{file_id: loaded_result}` with all files at or above the sample's hierarchy level. Results are cached so shared files are parsed only once. |
|
|
194
217
|
| `transforms` | `list[Callable[[dict], dict]] \| None` | Applied in order after loading. Each receives and returns the full sample dict. |
|
|
195
218
|
|
|
219
|
+
For hierarchical modalities, the ds-crawler file ID is the key in
|
|
220
|
+
`{file_id: loaded_result}`. If files at different ancestor levels use the same
|
|
221
|
+
ID, the deepest matching file wins. This gives calibration files natural
|
|
222
|
+
inheritance semantics: a root-level `intrinsics` file can apply to every sample,
|
|
223
|
+
while a scene- or camera-level `intrinsics` file with the same ID overrides it
|
|
224
|
+
for descendants. Use ds-crawler's `indexing.id.override` when the physical file
|
|
225
|
+
name is incidental, such as `calib.json` or a UUID, and you want a stable
|
|
226
|
+
semantic key like `intrinsics`, `extrinsics`, or `calibration`. Do not use an
|
|
227
|
+
override when multiple distinct files at the same hierarchy level should all be
|
|
228
|
+
returned.
|
|
229
|
+
|
|
196
230
|
#### Sample dict
|
|
197
231
|
|
|
198
232
|
`dataset[i]` returns:
|
|
@@ -357,10 +391,15 @@ Writer resolution uses the same module and function metadata:
|
|
|
357
391
|
|
|
358
392
|
## ds-crawler integration
|
|
359
393
|
|
|
360
|
-
Each logical modality must have its own ds-crawler index. Usually that means one modality root with its own `.ds_crawler` artifacts. When several logical modalities share one physical directory or zip, put each artifact set under `.ds_crawler/<metadata_scope>/` and set `Modality(..., metadata_scope=...)
|
|
394
|
+
Each logical modality must have its own ds-crawler index. Usually that means one modality root with its own `.ds_crawler` artifacts. When several logical modalities share one physical directory or zip, put each artifact set under `.ds_crawler/<metadata_scope>/` and set `Modality(..., metadata_scope=...)` or append `#scope=<metadata_scope>` to the path.
|
|
361
395
|
Files across regular modalities are matched by IDs from those indexes, so **the indexed hierarchy and naming conventions must be consistent** across modalities up to modality-specific parts captured in the config.
|
|
362
396
|
|
|
363
397
|
Calibration files or other per-scene/per-sequence metadata can be loaded via `hierarchical_modalities`. These files are matched to samples based on their position in the hierarchy — all files at or above a sample's hierarchy level are included and cached for efficiency.
|
|
398
|
+
Root-level calibration files, for example `calib.json` at the top of a shared
|
|
399
|
+
zip, should be indexed by ds-crawler without a hierarchy block. euler-loading
|
|
400
|
+
treats that root file as an ancestor of every sample. If the modality uses
|
|
401
|
+
`collapse_single=True`, the loaded calibration value is returned directly as
|
|
402
|
+
long as exactly one hierarchical file matches the sample.
|
|
364
403
|
|
|
365
404
|
## Layout-aware augmentation loading
|
|
366
405
|
|
|
@@ -484,6 +523,7 @@ All built-in loaders accept both filesystem paths (`str`) and in-memory buffers
|
|
|
484
523
|
| `reference_rgb` | Clear-weather reference RGB image as float32, normalised to [0, 1] |
|
|
485
524
|
| `semantic_segmentation` | Single-channel Cityscapes `labelIds` or `labelTrainIds` PNG as class IDs |
|
|
486
525
|
| `semantic_segmentation_color` | Cityscapes `labelColor` PNG as RGB uint8 labels |
|
|
526
|
+
| `sky_mask` | Binary mask from Cityscapes labels; uses sky `trainId=10` for `labelTrainIds`, `labelId=23` for `labelIds`, or RGB `meta["sky_class"]` such as `[0, 0, 23]` |
|
|
487
527
|
| `panoptic_segmentation` | COCO-style RGB panoptic PNG decoded to integer segment IDs |
|
|
488
528
|
| `lidar_point_cloud` / `point_cloud` | Lidar `.bin` file as `(N, 6)` float64: `x, y, z, intensity, ring, timestamp` |
|
|
489
529
|
| `sparse_depth` | Alias for MUSES lidar points for sparse-depth style workflows |
|
|
@@ -3,18 +3,21 @@ from __future__ import annotations
|
|
|
3
3
|
from collections.abc import Mapping
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from typing import Any
|
|
6
|
+
from urllib.parse import parse_qsl
|
|
6
7
|
|
|
7
8
|
from ds_crawler import (
|
|
8
9
|
get_dataset_contract,
|
|
10
|
+
list_metadata_scopes,
|
|
9
11
|
load_dataset_config,
|
|
10
12
|
load_dataset_split as _load_dataset_split,
|
|
11
13
|
validate_metadata_scope,
|
|
12
14
|
)
|
|
13
|
-
from ds_crawler.zip_utils import read_metadata_json
|
|
14
15
|
from ds_crawler.config import CONFIG_FILENAME as _DS_CRAWLER_CONFIG_FILENAME
|
|
15
16
|
from ds_crawler.zip_utils import (
|
|
17
|
+
DATASET_HEAD_FILENAME as _DS_CRAWLER_HEAD_FILENAME,
|
|
16
18
|
OUTPUT_FILENAME as _DS_CRAWLER_OUTPUT_FILENAME,
|
|
17
19
|
get_split_filename,
|
|
20
|
+
read_metadata_json,
|
|
18
21
|
validate_split_name,
|
|
19
22
|
)
|
|
20
23
|
|
|
@@ -63,7 +66,38 @@ def extract_ds_crawler_properties(index_output: Mapping[str, Any]) -> dict[str,
|
|
|
63
66
|
return {}
|
|
64
67
|
|
|
65
68
|
|
|
66
|
-
|
|
69
|
+
_METADATA_FILENAMES = (
|
|
70
|
+
_DS_CRAWLER_OUTPUT_FILENAME,
|
|
71
|
+
_DS_CRAWLER_CONFIG_FILENAME,
|
|
72
|
+
_DS_CRAWLER_HEAD_FILENAME,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _parse_path_with_scope(path: str) -> tuple[str, str | None]:
|
|
77
|
+
"""Extract an optional ``#scope=<name>`` selector from a path."""
|
|
78
|
+
candidate_path, sep, fragment = path.rpartition("#")
|
|
79
|
+
if not sep:
|
|
80
|
+
return path, None
|
|
81
|
+
|
|
82
|
+
params = parse_qsl(fragment, keep_blank_values=True)
|
|
83
|
+
scope_values = [value for key, value in params if key == "scope"]
|
|
84
|
+
if not scope_values:
|
|
85
|
+
return path, None
|
|
86
|
+
|
|
87
|
+
unsupported = [key for key, _ in params if key != "scope"]
|
|
88
|
+
if unsupported:
|
|
89
|
+
raise ValueError(
|
|
90
|
+
"Unsupported modality path fragment parameter(s): "
|
|
91
|
+
f"{', '.join(sorted(set(unsupported)))}. "
|
|
92
|
+
"Only '#scope=<metadata_scope>' is supported."
|
|
93
|
+
)
|
|
94
|
+
if len(scope_values) > 1:
|
|
95
|
+
raise ValueError("Modality path may contain at most one '#scope=' selector.")
|
|
96
|
+
|
|
97
|
+
return candidate_path, validate_metadata_scope(scope_values[0])
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _parse_path_with_split(path: str) -> tuple[str, str | None]:
|
|
67
101
|
"""Extract an optional inline split suffix from a colon-separated path.
|
|
68
102
|
|
|
69
103
|
Supports paths like ``/data/ds.zip:train`` where the part after the
|
|
@@ -88,16 +122,134 @@ def parse_path_with_split(path: str) -> tuple[str, str | None]:
|
|
|
88
122
|
return candidate_path, split
|
|
89
123
|
|
|
90
124
|
|
|
125
|
+
def parse_modality_path(path: str) -> tuple[str, str | None, str | None]:
|
|
126
|
+
"""Extract inline ``:split`` and ``#scope=...`` selectors from a path."""
|
|
127
|
+
parsed_path, metadata_scope = _parse_path_with_scope(path)
|
|
128
|
+
parsed_path, split = _parse_path_with_split(parsed_path)
|
|
129
|
+
return parsed_path, split, metadata_scope
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def parse_path_with_split(path: str) -> tuple[str, str | None]:
|
|
133
|
+
"""Extract an optional inline split suffix from a colon-separated path."""
|
|
134
|
+
parsed_path, split, _ = parse_modality_path(path)
|
|
135
|
+
return parsed_path, split
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _read_metadata_json_safe(
|
|
139
|
+
dataset_path: Path,
|
|
140
|
+
filename: str,
|
|
141
|
+
*,
|
|
142
|
+
metadata_scope: str | None = None,
|
|
143
|
+
) -> dict[str, Any] | None:
|
|
144
|
+
try:
|
|
145
|
+
return read_metadata_json(
|
|
146
|
+
dataset_path,
|
|
147
|
+
filename,
|
|
148
|
+
metadata_scope=metadata_scope,
|
|
149
|
+
)
|
|
150
|
+
except FileNotFoundError:
|
|
151
|
+
return None
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _has_root_metadata(dataset_path: Path) -> bool:
|
|
155
|
+
"""Return whether unscoped ds-crawler metadata exists at a dataset root."""
|
|
156
|
+
return any(
|
|
157
|
+
_read_metadata_json_safe(dataset_path, filename) is not None
|
|
158
|
+
for filename in _METADATA_FILENAMES
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
|
|
91
162
|
def _has_scoped_metadata(dataset_path: Path, metadata_scope: str) -> bool:
|
|
92
163
|
"""Return whether ds-crawler metadata exists under a configured scope."""
|
|
93
|
-
|
|
94
|
-
|
|
164
|
+
return any(
|
|
165
|
+
_read_metadata_json_safe(
|
|
95
166
|
dataset_path,
|
|
96
167
|
filename,
|
|
97
168
|
metadata_scope=metadata_scope,
|
|
98
|
-
) is not None
|
|
99
|
-
|
|
100
|
-
|
|
169
|
+
) is not None
|
|
170
|
+
for filename in _METADATA_FILENAMES
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _list_metadata_scopes_safe(dataset_path: Path) -> list[str]:
|
|
175
|
+
try:
|
|
176
|
+
return list_metadata_scopes(dataset_path)
|
|
177
|
+
except FileNotFoundError:
|
|
178
|
+
return []
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _read_scope_modality_key(dataset_path: Path, metadata_scope: str) -> str | None:
|
|
182
|
+
head = _read_metadata_json_safe(
|
|
183
|
+
dataset_path,
|
|
184
|
+
_DS_CRAWLER_HEAD_FILENAME,
|
|
185
|
+
metadata_scope=metadata_scope,
|
|
186
|
+
)
|
|
187
|
+
modality = head.get("modality") if isinstance(head, Mapping) else None
|
|
188
|
+
key = modality.get("key") if isinstance(modality, Mapping) else None
|
|
189
|
+
return as_non_empty_str(key)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def infer_metadata_scope(
|
|
193
|
+
path: str | Path,
|
|
194
|
+
*,
|
|
195
|
+
candidates: list[str | None],
|
|
196
|
+
) -> str | None:
|
|
197
|
+
"""Infer a ds-crawler metadata scope only when the choice is deterministic.
|
|
198
|
+
|
|
199
|
+
Legacy root-level metadata always wins for an unscoped modality. If no
|
|
200
|
+
root metadata exists, a single available scope is selected automatically.
|
|
201
|
+
With multiple scopes, a scope is selected only when exactly one scope name
|
|
202
|
+
or scoped ``dataset-head.json`` modality key matches the provided
|
|
203
|
+
candidates.
|
|
204
|
+
"""
|
|
205
|
+
dataset_path = Path(path)
|
|
206
|
+
if _has_root_metadata(dataset_path):
|
|
207
|
+
return None
|
|
208
|
+
|
|
209
|
+
scopes = _list_metadata_scopes_safe(dataset_path)
|
|
210
|
+
if not scopes:
|
|
211
|
+
return None
|
|
212
|
+
if len(scopes) == 1:
|
|
213
|
+
return scopes[0]
|
|
214
|
+
|
|
215
|
+
candidate_set = {
|
|
216
|
+
candidate
|
|
217
|
+
for candidate in (as_non_empty_str(value) for value in candidates)
|
|
218
|
+
if candidate is not None
|
|
219
|
+
}
|
|
220
|
+
if not candidate_set:
|
|
221
|
+
_raise_ambiguous_metadata_scope(path, scopes, candidate_set)
|
|
222
|
+
|
|
223
|
+
matches = {
|
|
224
|
+
scope
|
|
225
|
+
for scope in scopes
|
|
226
|
+
if scope in candidate_set
|
|
227
|
+
or _read_scope_modality_key(dataset_path, scope) in candidate_set
|
|
228
|
+
}
|
|
229
|
+
if len(matches) == 1:
|
|
230
|
+
return next(iter(matches))
|
|
231
|
+
if len(matches) > 1:
|
|
232
|
+
_raise_ambiguous_metadata_scope(path, sorted(matches), candidate_set)
|
|
233
|
+
|
|
234
|
+
_raise_ambiguous_metadata_scope(path, scopes, candidate_set)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _raise_ambiguous_metadata_scope(
|
|
238
|
+
path: str | Path,
|
|
239
|
+
scopes: list[str],
|
|
240
|
+
candidates: set[str],
|
|
241
|
+
) -> None:
|
|
242
|
+
candidate_text = (
|
|
243
|
+
f" Candidates were: {', '.join(sorted(candidates))}."
|
|
244
|
+
if candidates
|
|
245
|
+
else ""
|
|
246
|
+
)
|
|
247
|
+
raise ValueError(
|
|
248
|
+
f"Multiple ds-crawler metadata scopes are available for {path!s}: "
|
|
249
|
+
f"{', '.join(scopes)}.{candidate_text} "
|
|
250
|
+
"Specify Modality(..., metadata_scope=...) or append "
|
|
251
|
+
"'#scope=<metadata_scope>' to the path."
|
|
252
|
+
)
|
|
101
253
|
|
|
102
254
|
|
|
103
255
|
def load_dataset_config_for_scope(
|
|
@@ -31,8 +31,10 @@ except ImportError: # pragma: no cover - compatibility with older ds-crawler
|
|
|
31
31
|
_ds_crawler_get_layout_addon = None
|
|
32
32
|
|
|
33
33
|
from ._ds_crawler_utils import (
|
|
34
|
+
as_non_empty_str,
|
|
35
|
+
infer_metadata_scope,
|
|
34
36
|
load_index_output,
|
|
35
|
-
|
|
37
|
+
parse_modality_path,
|
|
36
38
|
validate_metadata_scope,
|
|
37
39
|
)
|
|
38
40
|
from ._metadata import _build_runlog_entry, _get_ds_crawler_descriptor
|
|
@@ -303,6 +305,9 @@ class Modality:
|
|
|
303
305
|
``output.json`` from a prior indexing run). A colon-separated
|
|
304
306
|
split suffix is also accepted (e.g. ``/data/ds.zip:train``);
|
|
305
307
|
the suffix is extracted and used as the ``split`` parameter.
|
|
308
|
+
A metadata scope may also be selected inline with
|
|
309
|
+
``#scope=<metadata_scope>``, including after a split suffix
|
|
310
|
+
(e.g. ``/data/ds.zip:train#scope=rgb``).
|
|
306
311
|
origin_path: Optional original path string before any copying or symlinking for i.e. slurm staging.
|
|
307
312
|
This is not used by euler-loading itself but can be useful for experiment
|
|
308
313
|
logging to retain references to the original dataset location.
|
|
@@ -337,9 +342,10 @@ class Modality:
|
|
|
337
342
|
logical modalities to share one physical directory or zip while
|
|
338
343
|
keeping separate ``dataset-head.json``, ``ds-crawler.json``,
|
|
339
344
|
``index.json``, and split files under
|
|
340
|
-
``.ds_crawler/<metadata_scope>/``.
|
|
341
|
-
|
|
342
|
-
|
|
345
|
+
``.ds_crawler/<metadata_scope>/``. If omitted, euler-loading
|
|
346
|
+
infers a scope only when the choice is deterministic. When
|
|
347
|
+
configured scoped artifacts are absent, loading falls back to
|
|
348
|
+
the legacy root-level ``.ds_crawler`` layout.
|
|
343
349
|
cache: Optional opt-in/out for in-memory caching of loaded values.
|
|
344
350
|
Only meaningful for hierarchical modalities; regular
|
|
345
351
|
modalities load once per sample and never cache.
|
|
@@ -377,7 +383,7 @@ class Modality:
|
|
|
377
383
|
metadata: dict[str, Any] = field(default_factory=dict)
|
|
378
384
|
|
|
379
385
|
def __post_init__(self) -> None:
|
|
380
|
-
parsed_path, path_split =
|
|
386
|
+
parsed_path, path_split, path_metadata_scope = parse_modality_path(self.path)
|
|
381
387
|
if path_split is not None:
|
|
382
388
|
if self.split is not None:
|
|
383
389
|
raise ValueError(
|
|
@@ -385,16 +391,93 @@ class Modality:
|
|
|
385
391
|
f"({path_split!r}) but an explicit split={self.split!r} "
|
|
386
392
|
f"was also provided. Use one or the other, not both."
|
|
387
393
|
)
|
|
394
|
+
|
|
395
|
+
normalized_metadata_scope = (
|
|
396
|
+
validate_metadata_scope(self.metadata_scope)
|
|
397
|
+
if self.metadata_scope is not None
|
|
398
|
+
else None
|
|
399
|
+
)
|
|
400
|
+
if path_metadata_scope is not None:
|
|
401
|
+
if (
|
|
402
|
+
normalized_metadata_scope is not None
|
|
403
|
+
and normalized_metadata_scope != path_metadata_scope
|
|
404
|
+
):
|
|
405
|
+
raise ValueError(
|
|
406
|
+
f"Modality path {self.path!r} contains an inline "
|
|
407
|
+
f"metadata scope ({path_metadata_scope!r}) but an "
|
|
408
|
+
f"explicit metadata_scope={self.metadata_scope!r} was "
|
|
409
|
+
"also provided. Use one metadata scope selector."
|
|
410
|
+
)
|
|
411
|
+
normalized_metadata_scope = path_metadata_scope
|
|
412
|
+
|
|
413
|
+
if parsed_path != self.path:
|
|
388
414
|
object.__setattr__(self, "path", parsed_path)
|
|
415
|
+
if path_split is not None:
|
|
389
416
|
object.__setattr__(self, "split", path_split)
|
|
390
|
-
if
|
|
417
|
+
if normalized_metadata_scope is not None:
|
|
391
418
|
object.__setattr__(
|
|
392
419
|
self,
|
|
393
420
|
"metadata_scope",
|
|
394
|
-
|
|
421
|
+
normalized_metadata_scope,
|
|
395
422
|
)
|
|
396
423
|
|
|
397
424
|
|
|
425
|
+
def _loader_modality_type(loader: Callable[..., Any] | None) -> str | None:
|
|
426
|
+
meta = getattr(loader, "_modality_meta", None)
|
|
427
|
+
if not isinstance(meta, Mapping):
|
|
428
|
+
return None
|
|
429
|
+
return as_non_empty_str(meta.get("type"))
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _metadata_modality_type(modality: Modality) -> str | None:
|
|
433
|
+
euler_loading = modality.metadata.get("euler_loading")
|
|
434
|
+
if not isinstance(euler_loading, Mapping):
|
|
435
|
+
return None
|
|
436
|
+
return as_non_empty_str(euler_loading.get("modality_type"))
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def _metadata_scope_candidates(name: str, modality: Modality) -> list[str | None]:
|
|
440
|
+
return [
|
|
441
|
+
name,
|
|
442
|
+
modality.modality_type,
|
|
443
|
+
_metadata_modality_type(modality),
|
|
444
|
+
_loader_modality_type(modality.loader),
|
|
445
|
+
]
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _resolve_metadata_scope(name: str, modality: Modality) -> Modality:
|
|
449
|
+
if modality.metadata_scope is not None:
|
|
450
|
+
return modality
|
|
451
|
+
|
|
452
|
+
metadata_scope = infer_metadata_scope(
|
|
453
|
+
modality.path,
|
|
454
|
+
candidates=_metadata_scope_candidates(name, modality),
|
|
455
|
+
)
|
|
456
|
+
if metadata_scope is None:
|
|
457
|
+
return modality
|
|
458
|
+
return replace(modality, metadata_scope=metadata_scope)
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def _resolve_metadata_scopes(
|
|
462
|
+
modalities: dict[str, Modality],
|
|
463
|
+
) -> dict[str, Modality]:
|
|
464
|
+
return {
|
|
465
|
+
name: _resolve_metadata_scope(name, modality)
|
|
466
|
+
for name, modality in modalities.items()
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def _modality_selector_suffix(modality: Modality) -> str:
|
|
471
|
+
selectors = []
|
|
472
|
+
if modality.split is not None:
|
|
473
|
+
selectors.append(f"split={modality.split}")
|
|
474
|
+
if modality.metadata_scope is not None:
|
|
475
|
+
selectors.append(f"metadata_scope={modality.metadata_scope}")
|
|
476
|
+
if not selectors:
|
|
477
|
+
return ""
|
|
478
|
+
return f" ({', '.join(selectors)})"
|
|
479
|
+
|
|
480
|
+
|
|
398
481
|
class MultiModalDataset(_BaseDataset):
|
|
399
482
|
"""PyTorch Dataset that loads synchronized multi-modal samples.
|
|
400
483
|
|
|
@@ -581,6 +664,7 @@ class MultiModalDataset(_BaseDataset):
|
|
|
581
664
|
if not modalities:
|
|
582
665
|
raise ValueError("At least one modality must be provided.")
|
|
583
666
|
|
|
667
|
+
modalities = _resolve_metadata_scopes(modalities)
|
|
584
668
|
indexed: dict[str, dict[str, Any]] = {}
|
|
585
669
|
layouts: dict[str, dict[str, Any] | None] = {}
|
|
586
670
|
for name, modality in modalities.items():
|
|
@@ -663,8 +747,12 @@ class MultiModalDataset(_BaseDataset):
|
|
|
663
747
|
if not modalities:
|
|
664
748
|
raise ValueError("At least one modality must be provided.")
|
|
665
749
|
|
|
750
|
+
modalities = _resolve_metadata_scopes(modalities)
|
|
751
|
+
hierarchical_modalities = _resolve_metadata_scopes(
|
|
752
|
+
hierarchical_modalities or {}
|
|
753
|
+
)
|
|
666
754
|
self._modalities = modalities
|
|
667
|
-
self._hierarchical_modalities = hierarchical_modalities
|
|
755
|
+
self._hierarchical_modalities = hierarchical_modalities
|
|
668
756
|
self._transforms = transforms or []
|
|
669
757
|
self._index_outputs: dict[str, dict[str, Any]] = {}
|
|
670
758
|
self._hierarchical_index_outputs: dict[str, dict[str, Any]] = {}
|
|
@@ -703,7 +791,7 @@ class MultiModalDataset(_BaseDataset):
|
|
|
703
791
|
name,
|
|
704
792
|
len(lookup),
|
|
705
793
|
modality.path,
|
|
706
|
-
|
|
794
|
+
_modality_selector_suffix(modality),
|
|
707
795
|
)
|
|
708
796
|
|
|
709
797
|
# -- Index hierarchical modalities -----------------------------------
|
|
@@ -738,7 +826,7 @@ class MultiModalDataset(_BaseDataset):
|
|
|
738
826
|
total_files,
|
|
739
827
|
len(files_by_level),
|
|
740
828
|
modality.path,
|
|
741
|
-
|
|
829
|
+
_modality_selector_suffix(modality),
|
|
742
830
|
)
|
|
743
831
|
|
|
744
832
|
# -- Compute common IDs (intersection across regular modalities) -----
|
|
@@ -12,6 +12,7 @@ Return types
|
|
|
12
12
|
containing raw Cityscapes ``labelIds`` or ``labelTrainIds``.
|
|
13
13
|
- **semantic_segmentation_color** -- ``np.ndarray`` of shape ``(H, W, 3)``
|
|
14
14
|
uint8 containing Cityscapes RGB colours.
|
|
15
|
+
- **sky_mask** -- ``np.ndarray`` of shape ``(H, W)`` bool.
|
|
15
16
|
- **panoptic_segmentation** -- ``np.ndarray`` of shape ``(H, W)`` int64
|
|
16
17
|
containing COCO-style panoptic segment IDs decoded from RGB.
|
|
17
18
|
- **lidar_point_cloud** / **sparse_depth** -- ``np.ndarray`` of shape
|
|
@@ -35,6 +36,8 @@ from PIL import Image
|
|
|
35
36
|
from euler_loading.loaders._annotations import modality_meta
|
|
36
37
|
|
|
37
38
|
_LIDAR_COLUMNS = ["x", "y", "z", "intensity", "ring", "timestamp"]
|
|
39
|
+
_SKY_LABEL_ID = 23
|
|
40
|
+
_SKY_TRAIN_ID = 10
|
|
38
41
|
|
|
39
42
|
|
|
40
43
|
def _load_image_rgb(path: Union[str, BinaryIO]) -> np.ndarray:
|
|
@@ -56,6 +59,59 @@ def _load_single_channel_labels(path: Union[str, BinaryIO]) -> np.ndarray:
|
|
|
56
59
|
return arr
|
|
57
60
|
|
|
58
61
|
|
|
62
|
+
def _select_sky_class_id(
|
|
63
|
+
path: Union[str, BinaryIO],
|
|
64
|
+
meta: Mapping[str, Any] | None,
|
|
65
|
+
attributes: Mapping[str, Any] | None,
|
|
66
|
+
) -> int:
|
|
67
|
+
for source in (attributes, meta):
|
|
68
|
+
if not isinstance(source, Mapping):
|
|
69
|
+
continue
|
|
70
|
+
value = source.get("sky_class_id")
|
|
71
|
+
if isinstance(value, (int, np.integer)):
|
|
72
|
+
return int(value)
|
|
73
|
+
|
|
74
|
+
name = str(getattr(path, "name", path)).lower()
|
|
75
|
+
if "labelids" in name:
|
|
76
|
+
return _SKY_LABEL_ID
|
|
77
|
+
return _SKY_TRAIN_ID
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _select_sky_class_color(
|
|
81
|
+
meta: Mapping[str, Any] | None,
|
|
82
|
+
attributes: Mapping[str, Any] | None,
|
|
83
|
+
) -> np.ndarray | None:
|
|
84
|
+
for source in (attributes, meta):
|
|
85
|
+
if not isinstance(source, Mapping):
|
|
86
|
+
continue
|
|
87
|
+
value = source.get("sky_class")
|
|
88
|
+
if value is None:
|
|
89
|
+
value = source.get("sky_mask")
|
|
90
|
+
if value is None:
|
|
91
|
+
continue
|
|
92
|
+
|
|
93
|
+
color = np.asarray(value, dtype=np.uint8)
|
|
94
|
+
if color.shape != (3,):
|
|
95
|
+
raise ValueError("MUSES sky_class must be a 3-element RGB value")
|
|
96
|
+
return color
|
|
97
|
+
return None
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _load_sky_mask_array(
|
|
101
|
+
path: Union[str, BinaryIO],
|
|
102
|
+
meta: Mapping[str, Any] | None,
|
|
103
|
+
attributes: Mapping[str, Any] | None,
|
|
104
|
+
) -> np.ndarray:
|
|
105
|
+
color = _select_sky_class_color(meta, attributes)
|
|
106
|
+
if color is not None:
|
|
107
|
+
with Image.open(path) as image:
|
|
108
|
+
arr = np.array(image.convert("RGB"), dtype=np.uint8)
|
|
109
|
+
return np.all(arr == color, axis=-1)
|
|
110
|
+
|
|
111
|
+
labels = _load_single_channel_labels(path)
|
|
112
|
+
return labels == _select_sky_class_id(path, meta, attributes)
|
|
113
|
+
|
|
114
|
+
|
|
59
115
|
def _load_rgb_uint8(path: Union[str, BinaryIO]) -> np.ndarray:
|
|
60
116
|
"""Load an RGB image without normalisation."""
|
|
61
117
|
with Image.open(path) as image:
|
|
@@ -246,6 +302,29 @@ def semantic_segmentation(path: Union[str, BinaryIO], meta: dict[str, Any] | Non
|
|
|
246
302
|
return _load_single_channel_labels(path)
|
|
247
303
|
|
|
248
304
|
|
|
305
|
+
@modality_meta(
|
|
306
|
+
modality_type="sky_mask",
|
|
307
|
+
dtype="bool",
|
|
308
|
+
shape="HW",
|
|
309
|
+
file_formats=[".png"],
|
|
310
|
+
meta={
|
|
311
|
+
"encoding": "single_channel",
|
|
312
|
+
"sky_class": [0, 0, _SKY_LABEL_ID],
|
|
313
|
+
"sky_label_id": _SKY_LABEL_ID,
|
|
314
|
+
"sky_train_id": _SKY_TRAIN_ID,
|
|
315
|
+
},
|
|
316
|
+
)
|
|
317
|
+
def sky_mask(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None, *, attributes: dict[str, Any] | None = None) -> np.ndarray:
|
|
318
|
+
"""Load a MUSES sky mask as an ``(H, W)`` bool array.
|
|
319
|
+
|
|
320
|
+
Reads a single-channel Cityscapes ``labelIds`` / ``labelTrainIds`` PNG, or
|
|
321
|
+
an RGB-encoded class image when ``sky_class`` is provided. Override the
|
|
322
|
+
default ID with ``sky_class_id``, or set ``sky_class`` to an RGB value such
|
|
323
|
+
as ``[0, 0, 23]``.
|
|
324
|
+
"""
|
|
325
|
+
return _load_sky_mask_array(path, meta, attributes)
|
|
326
|
+
|
|
327
|
+
|
|
249
328
|
@modality_meta(
|
|
250
329
|
modality_type="semantic_segmentation_color",
|
|
251
330
|
dtype="uint8",
|