euler-loading 2.9.0__tar.gz → 2.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {euler_loading-2.9.0 → euler_loading-2.10.0}/PKG-INFO +1 -1
- {euler_loading-2.9.0 → euler_loading-2.10.0}/README.md +46 -7
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/_ds_crawler_utils.py +92 -45
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/_metadata.py +15 -4
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/dataset.py +29 -1
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/cpu/muses.py +168 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/generate/loaders.json +39 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/gpu/muses.py +168 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/package-lock.json +1 -1
- {euler_loading-2.9.0 → euler_loading-2.10.0}/pyproject.toml +1 -1
- {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/test_dataset.py +115 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/test_loaders.py +92 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/.github/workflows/workflow.yml +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/.gitignore +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/docs/loader-attributes.md +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/__init__.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/_dataset_contract.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/_resolution.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/_writing.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/indexing.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/__init__.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/_annotations.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/_writer_utils.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/contracts.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/cpu/__init__.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/cpu/generic.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/cpu/generic_dense_depth.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/cpu/real_drive_sim.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/cpu/vkitti2.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/generate/__init__.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/generate/__main__.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/generic.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/gpu/__init__.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/gpu/generic.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/gpu/generic_dense_depth.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/gpu/real_drive_sim.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/gpu/vkitti2.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/muses.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/real_drive_sim.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/vkitti2.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/preprocessing.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/example.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/sample_rds.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/__init__.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/conftest.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/example_rds_calib.json +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/test_id_schema.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/test_indexing.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/test_preprocessing.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/test_real_dataset.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/test_writing.py +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/vkitti_cpu_example_output.json +0 -0
- {euler_loading-2.9.0 → euler_loading-2.10.0}/vkitti_gpu_example_output.json +0 -0
|
@@ -55,7 +55,7 @@ Frozen dataclass describing one data modality.
|
|
|
55
55
|
|
|
56
56
|
| Parameter | Type | Description |
|
|
57
57
|
|-----------|------|-------------|
|
|
58
|
-
| `path` | `str` | Absolute path to the modality root directory or `.zip` archive. Must contain
|
|
58
|
+
| `path` | `str` | Absolute path to the modality root directory or `.zip` archive. Must contain ds-crawler metadata, either at `.ds_crawler/` or under the configured `metadata_scope`. |
|
|
59
59
|
| `origin_path` | `str \| None` | Original path before copying/symlinking (e.g. for SLURM staging). Not used by euler-loading itself — useful for experiment logging to retain references to the original dataset location. |
|
|
60
60
|
| `loader` | `Callable[..., Any] \| None` | Receives the file path (or `BinaryIO` buffer for zip-backed modalities) and an optional `meta` dict. Returns loaded data. When `None`, the loader is resolved automatically from the ds-crawler index (see [Automatic loader resolution](#automatic-loader-resolution)). |
|
|
61
61
|
| `writer` | `Callable[..., Any] \| None` | Receives `(path, value, meta)` and writes modality data to disk. When `None`, euler-loading tries to resolve a built-in writer from ds-crawler metadata (`write_<function>` or `write_<suffix>` for `read_<suffix>`). |
|
|
@@ -65,6 +65,7 @@ Frozen dataclass describing one data modality.
|
|
|
65
65
|
| `hierarchy_scope` | `str \| None` | Optional scope label for hierarchical modalities (e.g. `scene_camera`). |
|
|
66
66
|
| `applies_to` | `list[str] \| None` | Optional list of regular modality names a hierarchical modality applies to. |
|
|
67
67
|
| `split` | `str \| None` | Optional inline split name. Loads `.ds_crawler/split_<name>.json` from the modality root (directory or zip) and overlays it on the normal ds-crawler metadata. |
|
|
68
|
+
| `metadata_scope` | `str \| None` | Optional namespace below `.ds_crawler`, e.g. `.ds_crawler/camera_extrinsics/index.json`. Use this when multiple logical modalities share one physical directory or zip. If the scoped artifacts are absent, loading falls back to the legacy root-level `.ds_crawler` layout. |
|
|
68
69
|
| `cache` | `bool \| None` | Opt-in/out for in-memory caching of decoded values (only meaningful for hierarchical modalities; regular modalities are never cached). `True` keeps every distinct loaded file in a process-lifetime dict. `False` re-reads on every access. `None` (default): hierarchical → `True` because small shared calibration files benefit. |
|
|
69
70
|
| `collapse_single` | `bool` | For hierarchical modalities only. When `True`, a sample that matches exactly one hierarchical file returns that loaded value directly instead of `{file_id: value}`. This is useful for shared GT tensors used by multiple augmentations of the same source sample. |
|
|
70
71
|
| `metadata` | `dict[str, Any]` | Optional arbitrary metadata. Keys under `metadata["euler_loading"]` are treated as euler-loading defaults. |
|
|
@@ -129,13 +130,50 @@ dataset = MultiModalDataset(
|
|
|
129
130
|
|
|
130
131
|
This works for both directory-backed and zip-backed modalities. The split file only replaces the `dataset` payload; top-level metadata such as dataset type and euler-loading loader hints still come from the canonical ds-crawler index.
|
|
131
132
|
|
|
133
|
+
### Scoped ds-crawler Metadata
|
|
134
|
+
|
|
135
|
+
When a single physical root or zip contains files for several logical modalities, keep one ds-crawler artifact set per modality under `.ds_crawler/<metadata_scope>/`:
|
|
136
|
+
|
|
137
|
+
```
|
|
138
|
+
muses.zip
|
|
139
|
+
calib.json
|
|
140
|
+
frames/...
|
|
141
|
+
lidar/...
|
|
142
|
+
.ds_crawler/rgb/dataset-head.json
|
|
143
|
+
.ds_crawler/rgb/ds-crawler.json
|
|
144
|
+
.ds_crawler/rgb/index.json
|
|
145
|
+
.ds_crawler/camera_extrinsics/dataset-head.json
|
|
146
|
+
.ds_crawler/camera_extrinsics/ds-crawler.json
|
|
147
|
+
.ds_crawler/camera_extrinsics/index.json
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Then point multiple modalities at the same `path` and select the appropriate metadata scope:
|
|
151
|
+
|
|
152
|
+
```python
|
|
153
|
+
dataset = MultiModalDataset(
|
|
154
|
+
modalities={
|
|
155
|
+
"rgb": Modality("/data/muses.zip", metadata_scope="rgb"),
|
|
156
|
+
"sparse_depth": Modality("/data/muses.zip", metadata_scope="sparse_depth"),
|
|
157
|
+
},
|
|
158
|
+
hierarchical_modalities={
|
|
159
|
+
"camera_extrinsics": Modality(
|
|
160
|
+
"/data/muses.zip",
|
|
161
|
+
metadata_scope="camera_extrinsics",
|
|
162
|
+
collapse_single=True,
|
|
163
|
+
),
|
|
164
|
+
},
|
|
165
|
+
)
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
This is an additive layout. Existing roots with `.ds_crawler/index.json` continue to load unchanged, and a scoped modality falls back to that legacy location when the scoped artifacts are not present.
|
|
169
|
+
|
|
132
170
|
### `MultiModalDataset.modality_paths()`
|
|
133
171
|
|
|
134
|
-
Returns a dict mapping each regular modality name to `{"path": ..., "origin_path": ...}` and includes `split` when configured.
|
|
172
|
+
Returns a dict mapping each regular modality name to `{"path": ..., "origin_path": ...}` and includes `split` and `metadata_scope` when configured.
|
|
135
173
|
|
|
136
174
|
### `MultiModalDataset.hierarchical_modality_paths()`
|
|
137
175
|
|
|
138
|
-
Returns a dict mapping each hierarchical modality name to `{"path": ..., "origin_path": ...}` and includes `split` when configured.
|
|
176
|
+
Returns a dict mapping each hierarchical modality name to `{"path": ..., "origin_path": ...}` and includes `split` and `metadata_scope` when configured.
|
|
139
177
|
|
|
140
178
|
### `MultiModalDataset.get_modality_metadata(modality_name)`
|
|
141
179
|
|
|
@@ -145,7 +183,7 @@ Returns the ds-crawler metadata dict for the given modality.
|
|
|
145
183
|
|
|
146
184
|
PyTorch `Dataset`. On construction it:
|
|
147
185
|
|
|
148
|
-
1.
|
|
186
|
+
1. Loads each modality's ds-crawler index from `.ds_crawler/<metadata_scope>/index.json` when `metadata_scope` is configured and present; otherwise uses the legacy root-level ds-crawler lookup.
|
|
149
187
|
2. Computes the **sorted intersection** of file IDs across all regular modalities.
|
|
150
188
|
3. Logs warnings for unmatched files; raises `ValueError` when the intersection is empty.
|
|
151
189
|
|
|
@@ -319,9 +357,8 @@ Writer resolution uses the same module and function metadata:
|
|
|
319
357
|
|
|
320
358
|
## ds-crawler integration
|
|
321
359
|
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
Files across modalities are matched by these IDs, so **the directory structure must be consistent** across modalities (identical hierarchy and naming conventions up to the modality-specific parts captured in the config).
|
|
360
|
+
Each logical modality must have its own ds-crawler index. Usually that means one modality root with its own `.ds_crawler` artifacts. When several logical modalities share one physical directory or zip, put each artifact set under `.ds_crawler/<metadata_scope>/` and set `Modality(..., metadata_scope=...)`.
|
|
361
|
+
Files across regular modalities are matched by IDs from those indexes, so **the indexed hierarchy and naming conventions must be consistent** across modalities up to modality-specific parts captured in the config.
|
|
325
362
|
|
|
326
363
|
Calibration files or other per-scene/per-sequence metadata can be loaded via `hierarchical_modalities`. These files are matched to samples based on their position in the hierarchy — all files at or above a sample's hierarchy level are included and cached for efficiency.
|
|
327
364
|
|
|
@@ -450,6 +487,8 @@ All built-in loaders accept both filesystem paths (`str`) and in-memory buffers
|
|
|
450
487
|
| `panoptic_segmentation` | COCO-style RGB panoptic PNG decoded to integer segment IDs |
|
|
451
488
|
| `lidar_point_cloud` / `point_cloud` | Lidar `.bin` file as `(N, 6)` float64: `x, y, z, intensity, ring, timestamp` |
|
|
452
489
|
| `sparse_depth` | Alias for MUSES lidar points for sparse-depth style workflows |
|
|
490
|
+
| `read_intrinsics` | RGB camera intrinsics from MUSES `calib.json` as a `(3, 3)` matrix; defaults to `sensor="rgb"` |
|
|
491
|
+
| `read_extrinsics` | Camera extrinsics from MUSES `calib.json` as a `(4, 4)` matrix; defaults to `transform="lidar2rgb"` for sparse-depth reprojection into RGB pixels |
|
|
453
492
|
|
|
454
493
|
### Generic Dense Depth (`euler_loading.loaders.gpu.generic_dense_depth`)
|
|
455
494
|
|
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import re
|
|
4
3
|
from collections.abc import Mapping
|
|
5
4
|
from pathlib import Path
|
|
6
5
|
from typing import Any
|
|
7
6
|
|
|
8
|
-
from ds_crawler import
|
|
7
|
+
from ds_crawler import (
|
|
8
|
+
get_dataset_contract,
|
|
9
|
+
load_dataset_config,
|
|
10
|
+
load_dataset_split as _load_dataset_split,
|
|
11
|
+
validate_metadata_scope,
|
|
12
|
+
)
|
|
9
13
|
from ds_crawler.zip_utils import read_metadata_json
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
14
|
+
from ds_crawler.config import CONFIG_FILENAME as _DS_CRAWLER_CONFIG_FILENAME
|
|
15
|
+
from ds_crawler.zip_utils import (
|
|
16
|
+
OUTPUT_FILENAME as _DS_CRAWLER_OUTPUT_FILENAME,
|
|
17
|
+
get_split_filename,
|
|
18
|
+
validate_split_name,
|
|
19
|
+
)
|
|
15
20
|
|
|
16
21
|
|
|
17
22
|
def as_non_empty_str(value: Any) -> str | None:
|
|
@@ -58,9 +63,6 @@ def extract_ds_crawler_properties(index_output: Mapping[str, Any]) -> dict[str,
|
|
|
58
63
|
return {}
|
|
59
64
|
|
|
60
65
|
|
|
61
|
-
_SPLIT_NAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
|
|
62
|
-
|
|
63
|
-
|
|
64
66
|
def parse_path_with_split(path: str) -> tuple[str, str | None]:
|
|
65
67
|
"""Extract an optional inline split suffix from a colon-separated path.
|
|
66
68
|
|
|
@@ -78,36 +80,55 @@ def parse_path_with_split(path: str) -> tuple[str, str | None]:
|
|
|
78
80
|
candidate_split = path[colon_pos + 1:]
|
|
79
81
|
candidate_path = path[:colon_pos]
|
|
80
82
|
|
|
81
|
-
|
|
83
|
+
try:
|
|
84
|
+
split = validate_split_name(candidate_split)
|
|
85
|
+
except ValueError:
|
|
82
86
|
return path, None
|
|
83
87
|
|
|
84
|
-
return candidate_path,
|
|
88
|
+
return candidate_path, split
|
|
85
89
|
|
|
86
90
|
|
|
87
|
-
def
|
|
88
|
-
"""
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
)
|
|
98
|
-
return normalized
|
|
91
|
+
def _has_scoped_metadata(dataset_path: Path, metadata_scope: str) -> bool:
|
|
92
|
+
"""Return whether ds-crawler metadata exists under a configured scope."""
|
|
93
|
+
for filename in (_DS_CRAWLER_OUTPUT_FILENAME, _DS_CRAWLER_CONFIG_FILENAME):
|
|
94
|
+
if read_metadata_json(
|
|
95
|
+
dataset_path,
|
|
96
|
+
filename,
|
|
97
|
+
metadata_scope=metadata_scope,
|
|
98
|
+
) is not None:
|
|
99
|
+
return True
|
|
100
|
+
return False
|
|
99
101
|
|
|
100
102
|
|
|
101
|
-
def
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
103
|
+
def load_dataset_config_for_scope(
|
|
104
|
+
path: str | Path,
|
|
105
|
+
*,
|
|
106
|
+
metadata_scope: str | None,
|
|
107
|
+
load_dataset_config_fn: Any = load_dataset_config,
|
|
108
|
+
fallback_to_legacy: bool = True,
|
|
109
|
+
) -> Any:
|
|
110
|
+
"""Load a ds-crawler config, delegating scoped layout handling to ds-crawler."""
|
|
111
|
+
dataset_path = Path(path)
|
|
112
|
+
if metadata_scope is None:
|
|
113
|
+
return load_dataset_config_fn({"path": str(path)})
|
|
114
|
+
|
|
115
|
+
normalized_scope = validate_metadata_scope(metadata_scope)
|
|
116
|
+
try:
|
|
117
|
+
return load_dataset_config_fn(
|
|
118
|
+
{"path": str(path)},
|
|
119
|
+
metadata_scope=normalized_scope,
|
|
120
|
+
)
|
|
121
|
+
except FileNotFoundError:
|
|
122
|
+
if fallback_to_legacy and not _has_scoped_metadata(dataset_path, normalized_scope):
|
|
123
|
+
return load_dataset_config_fn({"path": str(path)})
|
|
124
|
+
raise
|
|
105
125
|
|
|
106
126
|
|
|
107
127
|
def load_index_output(
|
|
108
128
|
path: str | Path,
|
|
109
129
|
*,
|
|
110
130
|
split: str | None,
|
|
131
|
+
metadata_scope: str | None = None,
|
|
111
132
|
index_dataset_from_path_fn: Any,
|
|
112
133
|
strict: bool = False,
|
|
113
134
|
save_index: bool = False,
|
|
@@ -115,12 +136,39 @@ def load_index_output(
|
|
|
115
136
|
) -> dict[str, Any]:
|
|
116
137
|
"""Load a ds-crawler output, optionally overlaying an inline split.
|
|
117
138
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
139
|
+
Scoped metadata is loaded through ds-crawler's native ``metadata_scope``
|
|
140
|
+
support. If a scope is configured but no scoped metadata exists at all,
|
|
141
|
+
euler-loading still falls back to the legacy root-level lookup for
|
|
142
|
+
backward compatibility with older datasets.
|
|
122
143
|
"""
|
|
123
144
|
dataset_path = Path(path)
|
|
145
|
+
normalized_scope = (
|
|
146
|
+
validate_metadata_scope(metadata_scope)
|
|
147
|
+
if metadata_scope is not None
|
|
148
|
+
else None
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
if normalized_scope is not None:
|
|
152
|
+
try:
|
|
153
|
+
if split is not None:
|
|
154
|
+
return _load_dataset_split(
|
|
155
|
+
path,
|
|
156
|
+
validate_split_name(split),
|
|
157
|
+
strict=strict,
|
|
158
|
+
save_index=save_index,
|
|
159
|
+
force_reindex=force_reindex,
|
|
160
|
+
metadata_scope=normalized_scope,
|
|
161
|
+
)
|
|
162
|
+
return index_dataset_from_path_fn(
|
|
163
|
+
path,
|
|
164
|
+
strict=strict,
|
|
165
|
+
save_index=save_index,
|
|
166
|
+
force_reindex=force_reindex,
|
|
167
|
+
metadata_scope=normalized_scope,
|
|
168
|
+
)
|
|
169
|
+
except FileNotFoundError:
|
|
170
|
+
if _has_scoped_metadata(dataset_path, normalized_scope):
|
|
171
|
+
raise
|
|
124
172
|
|
|
125
173
|
if split is None:
|
|
126
174
|
return index_dataset_from_path_fn(
|
|
@@ -131,19 +179,18 @@ def load_index_output(
|
|
|
131
179
|
)
|
|
132
180
|
|
|
133
181
|
normalized_split = validate_split_name(split)
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
pass
|
|
182
|
+
try:
|
|
183
|
+
return _load_dataset_split(
|
|
184
|
+
path,
|
|
185
|
+
normalized_split,
|
|
186
|
+
strict=strict,
|
|
187
|
+
save_index=save_index,
|
|
188
|
+
force_reindex=force_reindex,
|
|
189
|
+
)
|
|
190
|
+
except FileNotFoundError:
|
|
191
|
+
# Fallback for mocked tests or callers that provide synthetic index
|
|
192
|
+
# loaders without a full ds-crawler config on disk.
|
|
193
|
+
pass
|
|
147
194
|
|
|
148
195
|
base_output = index_dataset_from_path_fn(
|
|
149
196
|
path,
|
|
@@ -12,6 +12,7 @@ from ._ds_crawler_utils import (
|
|
|
12
12
|
extract_ds_crawler_properties,
|
|
13
13
|
first_non_empty,
|
|
14
14
|
first_non_empty_list,
|
|
15
|
+
load_dataset_config_for_scope,
|
|
15
16
|
)
|
|
16
17
|
|
|
17
18
|
if TYPE_CHECKING:
|
|
@@ -66,6 +67,8 @@ def _build_runlog_entry(
|
|
|
66
67
|
entry: dict[str, Any] = {"path": modality.path}
|
|
67
68
|
if modality.split is not None:
|
|
68
69
|
entry["split"] = modality.split
|
|
70
|
+
if modality.metadata_scope is not None:
|
|
71
|
+
entry["metadata_scope"] = modality.metadata_scope
|
|
69
72
|
if used_as is not None:
|
|
70
73
|
entry["used_as"] = used_as
|
|
71
74
|
if slot is not None:
|
|
@@ -125,22 +128,26 @@ def _resolve_euler_loading_property(
|
|
|
125
128
|
def _get_ds_crawler_descriptor(
|
|
126
129
|
*,
|
|
127
130
|
path: str,
|
|
131
|
+
metadata_scope: str | None = None,
|
|
128
132
|
index_output: Mapping[str, Any] | None,
|
|
129
133
|
cache: dict[str, dict[str, Any]],
|
|
130
134
|
load_dataset_config_fn: Callable[[dict[str, Any]], Any],
|
|
131
135
|
) -> dict[str, Any]:
|
|
132
|
-
|
|
133
|
-
|
|
136
|
+
cache_key = f"{path}::{metadata_scope or ''}"
|
|
137
|
+
if cache_key not in cache:
|
|
138
|
+
cache[cache_key] = _read_ds_crawler_descriptor(
|
|
134
139
|
path=path,
|
|
140
|
+
metadata_scope=metadata_scope,
|
|
135
141
|
index_output=index_output,
|
|
136
142
|
load_dataset_config_fn=load_dataset_config_fn,
|
|
137
143
|
)
|
|
138
|
-
return cache[
|
|
144
|
+
return cache[cache_key]
|
|
139
145
|
|
|
140
146
|
|
|
141
147
|
def _read_ds_crawler_descriptor(
|
|
142
148
|
*,
|
|
143
149
|
path: str,
|
|
150
|
+
metadata_scope: str | None,
|
|
144
151
|
index_output: Mapping[str, Any] | None,
|
|
145
152
|
load_dataset_config_fn: Callable[[dict[str, Any]], Any],
|
|
146
153
|
) -> dict[str, Any]:
|
|
@@ -164,7 +171,11 @@ def _read_ds_crawler_descriptor(
|
|
|
164
171
|
descriptor["hierarchy_regex"] = hierarchy_regex
|
|
165
172
|
|
|
166
173
|
try:
|
|
167
|
-
cfg =
|
|
174
|
+
cfg = load_dataset_config_for_scope(
|
|
175
|
+
path,
|
|
176
|
+
metadata_scope=metadata_scope,
|
|
177
|
+
load_dataset_config_fn=load_dataset_config_fn,
|
|
178
|
+
)
|
|
168
179
|
except Exception:
|
|
169
180
|
return descriptor
|
|
170
181
|
|
|
@@ -30,7 +30,11 @@ try:
|
|
|
30
30
|
except ImportError: # pragma: no cover - compatibility with older ds-crawler
|
|
31
31
|
_ds_crawler_get_layout_addon = None
|
|
32
32
|
|
|
33
|
-
from ._ds_crawler_utils import
|
|
33
|
+
from ._ds_crawler_utils import (
|
|
34
|
+
load_index_output,
|
|
35
|
+
parse_path_with_split,
|
|
36
|
+
validate_metadata_scope,
|
|
37
|
+
)
|
|
34
38
|
from ._metadata import _build_runlog_entry, _get_ds_crawler_descriptor
|
|
35
39
|
from ._resolution import (
|
|
36
40
|
_resolve_loader,
|
|
@@ -328,6 +332,14 @@ class Modality:
|
|
|
328
332
|
split: Optional inline split name. When set, euler-loading reads the
|
|
329
333
|
full ds-crawler metadata from the modality root and overlays the
|
|
330
334
|
dataset payload from ``.ds_crawler/split_<name>.json``.
|
|
335
|
+
metadata_scope: Optional subdirectory below ``.ds_crawler`` to use for
|
|
336
|
+
this modality's ds-crawler artifacts. This allows multiple
|
|
337
|
+
logical modalities to share one physical directory or zip while
|
|
338
|
+
keeping separate ``dataset-head.json``, ``ds-crawler.json``,
|
|
339
|
+
``index.json``, and split files under
|
|
340
|
+
``.ds_crawler/<metadata_scope>/``. When scoped artifacts are
|
|
341
|
+
absent, loading falls back to the legacy root-level
|
|
342
|
+
``.ds_crawler`` layout.
|
|
331
343
|
cache: Optional opt-in/out for in-memory caching of loaded values.
|
|
332
344
|
Only meaningful for hierarchical modalities; regular
|
|
333
345
|
modalities load once per sample and never cache.
|
|
@@ -359,6 +371,7 @@ class Modality:
|
|
|
359
371
|
hierarchy_scope: str | None = None
|
|
360
372
|
applies_to: list[str] | None = None
|
|
361
373
|
split: str | None = None
|
|
374
|
+
metadata_scope: str | None = None
|
|
362
375
|
cache: bool | None = None
|
|
363
376
|
collapse_single: bool = False
|
|
364
377
|
metadata: dict[str, Any] = field(default_factory=dict)
|
|
@@ -374,6 +387,12 @@ class Modality:
|
|
|
374
387
|
)
|
|
375
388
|
object.__setattr__(self, "path", parsed_path)
|
|
376
389
|
object.__setattr__(self, "split", path_split)
|
|
390
|
+
if self.metadata_scope is not None:
|
|
391
|
+
object.__setattr__(
|
|
392
|
+
self,
|
|
393
|
+
"metadata_scope",
|
|
394
|
+
validate_metadata_scope(self.metadata_scope),
|
|
395
|
+
)
|
|
377
396
|
|
|
378
397
|
|
|
379
398
|
class MultiModalDataset(_BaseDataset):
|
|
@@ -433,6 +452,8 @@ class MultiModalDataset(_BaseDataset):
|
|
|
433
452
|
entry: dict[str, str | None] = {"path": path, "origin_path": origin}
|
|
434
453
|
if mod.split is not None:
|
|
435
454
|
entry["split"] = mod.split
|
|
455
|
+
if mod.metadata_scope is not None:
|
|
456
|
+
entry["metadata_scope"] = mod.metadata_scope
|
|
436
457
|
res[name] = entry
|
|
437
458
|
return res
|
|
438
459
|
|
|
@@ -450,6 +471,8 @@ class MultiModalDataset(_BaseDataset):
|
|
|
450
471
|
entry: dict[str, str | None] = {"path": path, "origin_path": origin}
|
|
451
472
|
if mod.split is not None:
|
|
452
473
|
entry["split"] = mod.split
|
|
474
|
+
if mod.metadata_scope is not None:
|
|
475
|
+
entry["metadata_scope"] = mod.metadata_scope
|
|
453
476
|
res[name] = entry
|
|
454
477
|
return res
|
|
455
478
|
|
|
@@ -496,6 +519,7 @@ class MultiModalDataset(_BaseDataset):
|
|
|
496
519
|
for name, modality in self._modalities.items():
|
|
497
520
|
descriptor = _get_ds_crawler_descriptor(
|
|
498
521
|
path=modality.path,
|
|
522
|
+
metadata_scope=modality.metadata_scope,
|
|
499
523
|
index_output=self._index_outputs.get(name),
|
|
500
524
|
cache=descriptor_cache,
|
|
501
525
|
load_dataset_config_fn=load_dataset_config,
|
|
@@ -513,6 +537,7 @@ class MultiModalDataset(_BaseDataset):
|
|
|
513
537
|
for name, modality in self._hierarchical_modalities.items():
|
|
514
538
|
descriptor = _get_ds_crawler_descriptor(
|
|
515
539
|
path=modality.path,
|
|
540
|
+
metadata_scope=modality.metadata_scope,
|
|
516
541
|
index_output=self._hierarchical_index_outputs.get(name),
|
|
517
542
|
cache=descriptor_cache,
|
|
518
543
|
load_dataset_config_fn=load_dataset_config,
|
|
@@ -562,6 +587,7 @@ class MultiModalDataset(_BaseDataset):
|
|
|
562
587
|
index = load_index_output(
|
|
563
588
|
modality.path,
|
|
564
589
|
split=modality.split,
|
|
590
|
+
metadata_scope=modality.metadata_scope,
|
|
565
591
|
index_dataset_from_path_fn=index_dataset_from_path,
|
|
566
592
|
)
|
|
567
593
|
indexed[name] = index
|
|
@@ -653,6 +679,7 @@ class MultiModalDataset(_BaseDataset):
|
|
|
653
679
|
index = load_index_output(
|
|
654
680
|
modality.path,
|
|
655
681
|
split=modality.split,
|
|
682
|
+
metadata_scope=modality.metadata_scope,
|
|
656
683
|
index_dataset_from_path_fn=index_dataset_from_path,
|
|
657
684
|
)
|
|
658
685
|
self._index_outputs[name] = index
|
|
@@ -688,6 +715,7 @@ class MultiModalDataset(_BaseDataset):
|
|
|
688
715
|
index = load_index_output(
|
|
689
716
|
modality.path,
|
|
690
717
|
split=modality.split,
|
|
718
|
+
metadata_scope=modality.metadata_scope,
|
|
691
719
|
index_dataset_from_path_fn=index_dataset_from_path,
|
|
692
720
|
)
|
|
693
721
|
self._hierarchical_index_outputs[name] = index
|
|
@@ -16,11 +16,17 @@ Return types
|
|
|
16
16
|
containing COCO-style panoptic segment IDs decoded from RGB.
|
|
17
17
|
- **lidar_point_cloud** / **sparse_depth** -- ``np.ndarray`` of shape
|
|
18
18
|
``(N, 6)`` float64 with columns ``x, y, z, intensity, ring, timestamp``.
|
|
19
|
+
- **read_intrinsics** -- ``np.ndarray`` of shape ``(3, 3)`` float32 parsed
|
|
20
|
+
from ``calib.json``.
|
|
21
|
+
- **read_extrinsics** -- ``np.ndarray`` of shape ``(4, 4)`` float32 parsed
|
|
22
|
+
from ``calib.json``.
|
|
19
23
|
"""
|
|
20
24
|
|
|
21
25
|
from __future__ import annotations
|
|
22
26
|
|
|
27
|
+
import json
|
|
23
28
|
import os
|
|
29
|
+
from collections.abc import Mapping
|
|
24
30
|
from typing import Any, BinaryIO, Union
|
|
25
31
|
|
|
26
32
|
import numpy as np
|
|
@@ -78,6 +84,128 @@ def _load_lidar_array(path: Union[str, BinaryIO]) -> np.ndarray:
|
|
|
78
84
|
return arr.reshape((-1, 6))
|
|
79
85
|
|
|
80
86
|
|
|
87
|
+
def _load_json(path: Union[str, BinaryIO]) -> dict[str, Any]:
|
|
88
|
+
"""Load JSON from a file path or an in-memory buffer."""
|
|
89
|
+
if isinstance(path, (str, os.PathLike)):
|
|
90
|
+
with open(path) as f:
|
|
91
|
+
data = json.load(f)
|
|
92
|
+
else:
|
|
93
|
+
data = json.load(path)
|
|
94
|
+
if not isinstance(data, dict):
|
|
95
|
+
name = getattr(path, "name", path)
|
|
96
|
+
raise ValueError(f"MUSES calibration file {name!r} must contain a JSON object.")
|
|
97
|
+
return data
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _select_string(
|
|
101
|
+
*,
|
|
102
|
+
meta: Mapping[str, Any] | None,
|
|
103
|
+
attributes: Mapping[str, Any] | None,
|
|
104
|
+
keys: tuple[str, ...],
|
|
105
|
+
) -> str | None:
|
|
106
|
+
for source in (attributes, meta):
|
|
107
|
+
if not isinstance(source, Mapping):
|
|
108
|
+
continue
|
|
109
|
+
for key in keys:
|
|
110
|
+
value = source.get(key)
|
|
111
|
+
if isinstance(value, str) and value:
|
|
112
|
+
return value
|
|
113
|
+
return None
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _available_keys(section: Mapping[str, Any]) -> str:
|
|
117
|
+
return ", ".join(sorted(str(key) for key in section.keys()))
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _read_intrinsics_array(
|
|
121
|
+
path: Union[str, BinaryIO],
|
|
122
|
+
meta: Mapping[str, Any] | None,
|
|
123
|
+
attributes: Mapping[str, Any] | None,
|
|
124
|
+
) -> np.ndarray:
|
|
125
|
+
data = _load_json(path)
|
|
126
|
+
intrinsics = data.get("intrinsics")
|
|
127
|
+
if not isinstance(intrinsics, Mapping):
|
|
128
|
+
raise ValueError("MUSES calibration JSON must contain an 'intrinsics' object.")
|
|
129
|
+
|
|
130
|
+
sensor = _select_string(
|
|
131
|
+
meta=meta,
|
|
132
|
+
attributes=attributes,
|
|
133
|
+
keys=("sensor", "camera", "intrinsics_sensor", "target_sensor"),
|
|
134
|
+
) or "rgb"
|
|
135
|
+
entry = intrinsics.get(sensor)
|
|
136
|
+
if not isinstance(entry, Mapping) or "K" not in entry:
|
|
137
|
+
raise KeyError(
|
|
138
|
+
f"MUSES calibration has no intrinsics for sensor {sensor!r}. "
|
|
139
|
+
f"Available sensors: {_available_keys(intrinsics)}"
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
K = np.asarray(entry["K"], dtype=np.float32)
|
|
143
|
+
if K.shape != (3, 3):
|
|
144
|
+
raise ValueError(
|
|
145
|
+
f"MUSES intrinsics for sensor {sensor!r} must have shape (3, 3), "
|
|
146
|
+
f"got {K.shape}."
|
|
147
|
+
)
|
|
148
|
+
return K
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _select_extrinsics_key(
|
|
152
|
+
*,
|
|
153
|
+
meta: Mapping[str, Any] | None,
|
|
154
|
+
attributes: Mapping[str, Any] | None,
|
|
155
|
+
) -> str:
|
|
156
|
+
explicit = _select_string(
|
|
157
|
+
meta=meta,
|
|
158
|
+
attributes=attributes,
|
|
159
|
+
keys=("transform", "extrinsics_key", "extrinsic", "key"),
|
|
160
|
+
)
|
|
161
|
+
if explicit is not None:
|
|
162
|
+
return explicit
|
|
163
|
+
|
|
164
|
+
source_sensor = _select_string(
|
|
165
|
+
meta=meta,
|
|
166
|
+
attributes=attributes,
|
|
167
|
+
keys=("source_sensor", "from_sensor", "source"),
|
|
168
|
+
)
|
|
169
|
+
target_sensor = _select_string(
|
|
170
|
+
meta=meta,
|
|
171
|
+
attributes=attributes,
|
|
172
|
+
keys=("target_sensor", "to_sensor", "target", "camera"),
|
|
173
|
+
)
|
|
174
|
+
if source_sensor is not None and target_sensor is not None:
|
|
175
|
+
return f"{source_sensor}2{target_sensor}"
|
|
176
|
+
|
|
177
|
+
return "lidar2rgb"
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _read_extrinsics_array(
|
|
181
|
+
path: Union[str, BinaryIO],
|
|
182
|
+
meta: Mapping[str, Any] | None,
|
|
183
|
+
attributes: Mapping[str, Any] | None,
|
|
184
|
+
) -> np.ndarray:
|
|
185
|
+
data = _load_json(path)
|
|
186
|
+
extrinsics = data.get("extrinsics")
|
|
187
|
+
if not isinstance(extrinsics, Mapping):
|
|
188
|
+
raise ValueError("MUSES calibration JSON must contain an 'extrinsics' object.")
|
|
189
|
+
|
|
190
|
+
transform = _select_extrinsics_key(meta=meta, attributes=attributes)
|
|
191
|
+
raw = extrinsics.get(transform)
|
|
192
|
+
if raw is None:
|
|
193
|
+
raise KeyError(
|
|
194
|
+
f"MUSES calibration has no extrinsics transform {transform!r}. "
|
|
195
|
+
f"Available transforms: {_available_keys(extrinsics)}"
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
T = np.asarray(raw, dtype=np.float32)
|
|
199
|
+
if T.shape == (3, 4):
|
|
200
|
+
T = np.vstack([T, np.array([[0.0, 0.0, 0.0, 1.0]], dtype=np.float32)])
|
|
201
|
+
if T.shape != (4, 4):
|
|
202
|
+
raise ValueError(
|
|
203
|
+
f"MUSES extrinsics transform {transform!r} must have shape "
|
|
204
|
+
f"(4, 4) or (3, 4), got {T.shape}."
|
|
205
|
+
)
|
|
206
|
+
return T
|
|
207
|
+
|
|
208
|
+
|
|
81
209
|
@modality_meta(
|
|
82
210
|
modality_type="rgb",
|
|
83
211
|
dtype="float32",
|
|
@@ -178,3 +306,43 @@ def point_cloud(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None,
|
|
|
178
306
|
def sparse_depth(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None, *, attributes: dict[str, Any] | None = None) -> np.ndarray:
|
|
179
307
|
"""Load MUSES lidar points for sparse-depth style workflows."""
|
|
180
308
|
return _load_lidar_array(path)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
@modality_meta(
|
|
312
|
+
modality_type="intrinsics",
|
|
313
|
+
dtype="float32",
|
|
314
|
+
hierarchical=True,
|
|
315
|
+
shape="3x3",
|
|
316
|
+
file_formats=[".json"],
|
|
317
|
+
meta={"default_sensor": "rgb"},
|
|
318
|
+
)
|
|
319
|
+
def read_intrinsics(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None, *, attributes: dict[str, Any] | None = None) -> np.ndarray:
|
|
320
|
+
"""Load a MUSES camera intrinsics matrix from ``calib.json``.
|
|
321
|
+
|
|
322
|
+
The default sensor is ``"rgb"``. Override it with ``meta`` or per-file
|
|
323
|
+
``attributes`` using ``sensor``, ``camera``, ``intrinsics_sensor``, or
|
|
324
|
+
``target_sensor``.
|
|
325
|
+
"""
|
|
326
|
+
return _read_intrinsics_array(path, meta, attributes)
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
@modality_meta(
|
|
330
|
+
modality_type="camera_extrinsics",
|
|
331
|
+
dtype="float32",
|
|
332
|
+
hierarchical=True,
|
|
333
|
+
shape="4x4",
|
|
334
|
+
file_formats=[".json"],
|
|
335
|
+
meta={
|
|
336
|
+
"default_transform": "lidar2rgb",
|
|
337
|
+
"transform_direction": "source_sensor_to_target_sensor",
|
|
338
|
+
},
|
|
339
|
+
)
|
|
340
|
+
def read_extrinsics(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None, *, attributes: dict[str, Any] | None = None) -> np.ndarray:
|
|
341
|
+
"""Load a MUSES camera extrinsics matrix from ``calib.json``.
|
|
342
|
+
|
|
343
|
+
The default transform is ``"lidar2rgb"``, suitable for projecting MUSES
|
|
344
|
+
lidar/sparse-depth points into the RGB camera. Override it with ``meta`` or
|
|
345
|
+
per-file ``attributes`` using ``transform`` / ``extrinsics_key`` or
|
|
346
|
+
``source_sensor`` + ``target_sensor``.
|
|
347
|
+
"""
|
|
348
|
+
return _read_extrinsics_array(path, meta, attributes)
|