euler-loading 2.9.0__tar.gz → 2.10.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. {euler_loading-2.9.0 → euler_loading-2.10.0}/PKG-INFO +1 -1
  2. {euler_loading-2.9.0 → euler_loading-2.10.0}/README.md +46 -7
  3. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/_ds_crawler_utils.py +92 -45
  4. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/_metadata.py +15 -4
  5. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/dataset.py +29 -1
  6. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/cpu/muses.py +168 -0
  7. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/generate/loaders.json +39 -0
  8. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/gpu/muses.py +168 -0
  9. {euler_loading-2.9.0 → euler_loading-2.10.0}/package-lock.json +1 -1
  10. {euler_loading-2.9.0 → euler_loading-2.10.0}/pyproject.toml +1 -1
  11. {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/test_dataset.py +115 -0
  12. {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/test_loaders.py +92 -0
  13. {euler_loading-2.9.0 → euler_loading-2.10.0}/.github/workflows/workflow.yml +0 -0
  14. {euler_loading-2.9.0 → euler_loading-2.10.0}/.gitignore +0 -0
  15. {euler_loading-2.9.0 → euler_loading-2.10.0}/docs/loader-attributes.md +0 -0
  16. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/__init__.py +0 -0
  17. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/_dataset_contract.py +0 -0
  18. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/_resolution.py +0 -0
  19. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/_writing.py +0 -0
  20. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/indexing.py +0 -0
  21. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/__init__.py +0 -0
  22. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/_annotations.py +0 -0
  23. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/_writer_utils.py +0 -0
  24. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/contracts.py +0 -0
  25. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/cpu/__init__.py +0 -0
  26. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/cpu/generic.py +0 -0
  27. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/cpu/generic_dense_depth.py +0 -0
  28. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/cpu/real_drive_sim.py +0 -0
  29. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/cpu/vkitti2.py +0 -0
  30. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/generate/__init__.py +0 -0
  31. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/generate/__main__.py +0 -0
  32. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/generic.py +0 -0
  33. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/gpu/__init__.py +0 -0
  34. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/gpu/generic.py +0 -0
  35. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/gpu/generic_dense_depth.py +0 -0
  36. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/gpu/real_drive_sim.py +0 -0
  37. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/gpu/vkitti2.py +0 -0
  38. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/muses.py +0 -0
  39. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/real_drive_sim.py +0 -0
  40. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/loaders/vkitti2.py +0 -0
  41. {euler_loading-2.9.0 → euler_loading-2.10.0}/euler_loading/preprocessing.py +0 -0
  42. {euler_loading-2.9.0 → euler_loading-2.10.0}/example.py +0 -0
  43. {euler_loading-2.9.0 → euler_loading-2.10.0}/sample_rds.py +0 -0
  44. {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/__init__.py +0 -0
  45. {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/conftest.py +0 -0
  46. {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/example_rds_calib.json +0 -0
  47. {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/test_id_schema.py +0 -0
  48. {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/test_indexing.py +0 -0
  49. {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/test_preprocessing.py +0 -0
  50. {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/test_real_dataset.py +0 -0
  51. {euler_loading-2.9.0 → euler_loading-2.10.0}/tests/test_writing.py +0 -0
  52. {euler_loading-2.9.0 → euler_loading-2.10.0}/vkitti_cpu_example_output.json +0 -0
  53. {euler_loading-2.9.0 → euler_loading-2.10.0}/vkitti_gpu_example_output.json +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: euler-loading
3
- Version: 2.9.0
3
+ Version: 2.10.0
4
4
  Summary: Multi-modal PyTorch dataloader using ds-crawler indices
5
5
  Requires-Python: >=3.9
6
6
  Requires-Dist: ds-crawler>=2.3.0
@@ -55,7 +55,7 @@ Frozen dataclass describing one data modality.
55
55
 
56
56
  | Parameter | Type | Description |
57
57
  |-----------|------|-------------|
58
- | `path` | `str` | Absolute path to the modality root directory or `.zip` archive. Must contain a `ds-crawler.config` or cached `output.json`. |
58
+ | `path` | `str` | Absolute path to the modality root directory or `.zip` archive. Must contain ds-crawler metadata, either at `.ds_crawler/` or under the configured `metadata_scope`. |
59
59
  | `origin_path` | `str \| None` | Original path before copying/symlinking (e.g. for SLURM staging). Not used by euler-loading itself — useful for experiment logging to retain references to the original dataset location. |
60
60
  | `loader` | `Callable[..., Any] \| None` | Receives the file path (or `BinaryIO` buffer for zip-backed modalities) and an optional `meta` dict. Returns loaded data. When `None`, the loader is resolved automatically from the ds-crawler index (see [Automatic loader resolution](#automatic-loader-resolution)). |
61
61
  | `writer` | `Callable[..., Any] \| None` | Receives `(path, value, meta)` and writes modality data to disk. When `None`, euler-loading tries to resolve a built-in writer from ds-crawler metadata (`write_<function>` or `write_<suffix>` for `read_<suffix>`). |
@@ -65,6 +65,7 @@ Frozen dataclass describing one data modality.
65
65
  | `hierarchy_scope` | `str \| None` | Optional scope label for hierarchical modalities (e.g. `scene_camera`). |
66
66
  | `applies_to` | `list[str] \| None` | Optional list of regular modality names a hierarchical modality applies to. |
67
67
  | `split` | `str \| None` | Optional inline split name. Loads `.ds_crawler/split_<name>.json` from the modality root (directory or zip) and overlays it on the normal ds-crawler metadata. |
68
+ | `metadata_scope` | `str \| None` | Optional namespace below `.ds_crawler`, e.g. `.ds_crawler/camera_extrinsics/index.json`. Use this when multiple logical modalities share one physical directory or zip. If the scoped artifacts are absent, loading falls back to the legacy root-level `.ds_crawler` layout. |
68
69
  | `cache` | `bool \| None` | Opt-in/out for in-memory caching of decoded values (only meaningful for hierarchical modalities; regular modalities are never cached). `True` keeps every distinct loaded file in a process-lifetime dict. `False` re-reads on every access. `None` (default): hierarchical → `True` because small shared calibration files benefit. |
69
70
  | `collapse_single` | `bool` | For hierarchical modalities only. When `True`, a sample that matches exactly one hierarchical file returns that loaded value directly instead of `{file_id: value}`. This is useful for shared GT tensors used by multiple augmentations of the same source sample. |
70
71
  | `metadata` | `dict[str, Any]` | Optional arbitrary metadata. Keys under `metadata["euler_loading"]` are treated as euler-loading defaults. |
@@ -129,13 +130,50 @@ dataset = MultiModalDataset(
129
130
 
130
131
  This works for both directory-backed and zip-backed modalities. The split file only replaces the `dataset` payload; top-level metadata such as dataset type and euler-loading loader hints still come from the canonical ds-crawler index.
131
132
 
133
+ ### Scoped ds-crawler Metadata
134
+
135
+ When a single physical root or zip contains files for several logical modalities, keep one ds-crawler artifact set per modality under `.ds_crawler/<metadata_scope>/`:
136
+
137
+ ```
138
+ muses.zip
139
+ calib.json
140
+ frames/...
141
+ lidar/...
142
+ .ds_crawler/rgb/dataset-head.json
143
+ .ds_crawler/rgb/ds-crawler.json
144
+ .ds_crawler/rgb/index.json
145
+ .ds_crawler/camera_extrinsics/dataset-head.json
146
+ .ds_crawler/camera_extrinsics/ds-crawler.json
147
+ .ds_crawler/camera_extrinsics/index.json
148
+ ```
149
+
150
+ Then point multiple modalities at the same `path` and select the appropriate metadata scope:
151
+
152
+ ```python
153
+ dataset = MultiModalDataset(
154
+ modalities={
155
+ "rgb": Modality("/data/muses.zip", metadata_scope="rgb"),
156
+ "sparse_depth": Modality("/data/muses.zip", metadata_scope="sparse_depth"),
157
+ },
158
+ hierarchical_modalities={
159
+ "camera_extrinsics": Modality(
160
+ "/data/muses.zip",
161
+ metadata_scope="camera_extrinsics",
162
+ collapse_single=True,
163
+ ),
164
+ },
165
+ )
166
+ ```
167
+
168
+ This is an additive layout. Existing roots with `.ds_crawler/index.json` continue to load unchanged, and a scoped modality falls back to that legacy location when the scoped artifacts are not present.
169
+
132
170
  ### `MultiModalDataset.modality_paths()`
133
171
 
134
- Returns a dict mapping each regular modality name to `{"path": ..., "origin_path": ...}` and includes `split` when configured.
172
+ Returns a dict mapping each regular modality name to `{"path": ..., "origin_path": ...}` and includes `split` and `metadata_scope` when configured.
135
173
 
136
174
  ### `MultiModalDataset.hierarchical_modality_paths()`
137
175
 
138
- Returns a dict mapping each hierarchical modality name to `{"path": ..., "origin_path": ...}` and includes `split` when configured.
176
+ Returns a dict mapping each hierarchical modality name to `{"path": ..., "origin_path": ...}` and includes `split` and `metadata_scope` when configured.
139
177
 
140
178
  ### `MultiModalDataset.get_modality_metadata(modality_name)`
141
179
 
@@ -145,7 +183,7 @@ Returns the ds-crawler metadata dict for the given modality.
145
183
 
146
184
  PyTorch `Dataset`. On construction it:
147
185
 
148
- 1. Runs `ds_crawler.index_dataset_from_path()` for every modality (regular and hierarchical).
186
+ 1. Loads each modality's ds-crawler index from `.ds_crawler/<metadata_scope>/index.json` when `metadata_scope` is configured and present; otherwise uses the legacy root-level ds-crawler lookup.
149
187
  2. Computes the **sorted intersection** of file IDs across all regular modalities.
150
188
  3. Logs warnings for unmatched files; raises `ValueError` when the intersection is empty.
151
189
 
@@ -319,9 +357,8 @@ Writer resolution uses the same module and function metadata:
319
357
 
320
358
  ## ds-crawler integration
321
359
 
322
- Every modality root must be independently indexable by ds-crawler.
323
- Place a `ds-crawler.config` in the root of each modality directory (or zip archive) ds-crawler will then parse the directory tree and assign each file an ID derived from its path properties.
324
- Files across modalities are matched by these IDs, so **the directory structure must be consistent** across modalities (identical hierarchy and naming conventions up to the modality-specific parts captured in the config).
360
+ Each logical modality must have its own ds-crawler index. Usually that means one modality root with its own `.ds_crawler` artifacts. When several logical modalities share one physical directory or zip, put each artifact set under `.ds_crawler/<metadata_scope>/` and set `Modality(..., metadata_scope=...)`.
361
+ Files across regular modalities are matched by IDs from those indexes, so **the indexed hierarchy and naming conventions must be consistent** across modalities up to modality-specific parts captured in the config.
325
362
 
326
363
  Calibration files or other per-scene/per-sequence metadata can be loaded via `hierarchical_modalities`. These files are matched to samples based on their position in the hierarchy — all files at or above a sample's hierarchy level are included and cached for efficiency.
327
364
 
@@ -450,6 +487,8 @@ All built-in loaders accept both filesystem paths (`str`) and in-memory buffers
450
487
  | `panoptic_segmentation` | COCO-style RGB panoptic PNG decoded to integer segment IDs |
451
488
  | `lidar_point_cloud` / `point_cloud` | Lidar `.bin` file as `(N, 6)` float64: `x, y, z, intensity, ring, timestamp` |
452
489
  | `sparse_depth` | Alias for MUSES lidar points for sparse-depth style workflows |
490
+ | `read_intrinsics` | RGB camera intrinsics from MUSES `calib.json` as a `(3, 3)` matrix; defaults to `sensor="rgb"` |
491
+ | `read_extrinsics` | Camera extrinsics from MUSES `calib.json` as a `(4, 4)` matrix; defaults to `transform="lidar2rgb"` for sparse-depth reprojection into RGB pixels |
453
492
 
454
493
  ### Generic Dense Depth (`euler_loading.loaders.gpu.generic_dense_depth`)
455
494
 
@@ -1,17 +1,22 @@
1
1
  from __future__ import annotations
2
2
 
3
- import re
4
3
  from collections.abc import Mapping
5
4
  from pathlib import Path
6
5
  from typing import Any
7
6
 
8
- from ds_crawler import get_dataset_contract
7
+ from ds_crawler import (
8
+ get_dataset_contract,
9
+ load_dataset_config,
10
+ load_dataset_split as _load_dataset_split,
11
+ validate_metadata_scope,
12
+ )
9
13
  from ds_crawler.zip_utils import read_metadata_json
10
-
11
- try:
12
- from ds_crawler import load_dataset_split as _load_dataset_split
13
- except ImportError:
14
- _load_dataset_split = None
14
+ from ds_crawler.config import CONFIG_FILENAME as _DS_CRAWLER_CONFIG_FILENAME
15
+ from ds_crawler.zip_utils import (
16
+ OUTPUT_FILENAME as _DS_CRAWLER_OUTPUT_FILENAME,
17
+ get_split_filename,
18
+ validate_split_name,
19
+ )
15
20
 
16
21
 
17
22
  def as_non_empty_str(value: Any) -> str | None:
@@ -58,9 +63,6 @@ def extract_ds_crawler_properties(index_output: Mapping[str, Any]) -> dict[str,
58
63
  return {}
59
64
 
60
65
 
61
- _SPLIT_NAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
62
-
63
-
64
66
  def parse_path_with_split(path: str) -> tuple[str, str | None]:
65
67
  """Extract an optional inline split suffix from a colon-separated path.
66
68
 
@@ -78,36 +80,55 @@ def parse_path_with_split(path: str) -> tuple[str, str | None]:
78
80
  candidate_split = path[colon_pos + 1:]
79
81
  candidate_path = path[:colon_pos]
80
82
 
81
- if not candidate_split or not _SPLIT_NAME_PATTERN.match(candidate_split):
83
+ try:
84
+ split = validate_split_name(candidate_split)
85
+ except ValueError:
82
86
  return path, None
83
87
 
84
- return candidate_path, candidate_split
88
+ return candidate_path, split
85
89
 
86
90
 
87
- def validate_split_name(split_name: str) -> str:
88
- """Validate and normalize an inline split name."""
89
- if not isinstance(split_name, str):
90
- raise ValueError("split_name must be a string")
91
- normalized = split_name.strip()
92
- if not normalized:
93
- raise ValueError("split_name must be a non-empty string")
94
- if not _SPLIT_NAME_PATTERN.match(normalized):
95
- raise ValueError(
96
- "split_name may only contain letters, digits, '.', '-', or '_'"
97
- )
98
- return normalized
91
+ def _has_scoped_metadata(dataset_path: Path, metadata_scope: str) -> bool:
92
+ """Return whether ds-crawler metadata exists under a configured scope."""
93
+ for filename in (_DS_CRAWLER_OUTPUT_FILENAME, _DS_CRAWLER_CONFIG_FILENAME):
94
+ if read_metadata_json(
95
+ dataset_path,
96
+ filename,
97
+ metadata_scope=metadata_scope,
98
+ ) is not None:
99
+ return True
100
+ return False
99
101
 
100
102
 
101
- def get_split_filename(split_name: str) -> str:
102
- """Return the ds-crawler metadata filename for *split_name*."""
103
- normalized = validate_split_name(split_name)
104
- return f"split_{normalized}.json"
103
+ def load_dataset_config_for_scope(
104
+ path: str | Path,
105
+ *,
106
+ metadata_scope: str | None,
107
+ load_dataset_config_fn: Any = load_dataset_config,
108
+ fallback_to_legacy: bool = True,
109
+ ) -> Any:
110
+ """Load a ds-crawler config, delegating scoped layout handling to ds-crawler."""
111
+ dataset_path = Path(path)
112
+ if metadata_scope is None:
113
+ return load_dataset_config_fn({"path": str(path)})
114
+
115
+ normalized_scope = validate_metadata_scope(metadata_scope)
116
+ try:
117
+ return load_dataset_config_fn(
118
+ {"path": str(path)},
119
+ metadata_scope=normalized_scope,
120
+ )
121
+ except FileNotFoundError:
122
+ if fallback_to_legacy and not _has_scoped_metadata(dataset_path, normalized_scope):
123
+ return load_dataset_config_fn({"path": str(path)})
124
+ raise
105
125
 
106
126
 
107
127
  def load_index_output(
108
128
  path: str | Path,
109
129
  *,
110
130
  split: str | None,
131
+ metadata_scope: str | None = None,
111
132
  index_dataset_from_path_fn: Any,
112
133
  strict: bool = False,
113
134
  save_index: bool = False,
@@ -115,12 +136,39 @@ def load_index_output(
115
136
  ) -> dict[str, Any]:
116
137
  """Load a ds-crawler output, optionally overlaying an inline split.
117
138
 
118
- When the installed ds-crawler version exposes ``load_dataset_split()``, it
119
- is used directly. Older versions are supported by loading the canonical
120
- output via ``index_dataset_from_path()`` and replacing ``output["dataset"]``
121
- with the contents of ``.ds_crawler/split_<name>.json``.
139
+ Scoped metadata is loaded through ds-crawler's native ``metadata_scope``
140
+ support. If a scope is configured but no scoped metadata exists at all,
141
+ euler-loading still falls back to the legacy root-level lookup for
142
+ backward compatibility with older datasets.
122
143
  """
123
144
  dataset_path = Path(path)
145
+ normalized_scope = (
146
+ validate_metadata_scope(metadata_scope)
147
+ if metadata_scope is not None
148
+ else None
149
+ )
150
+
151
+ if normalized_scope is not None:
152
+ try:
153
+ if split is not None:
154
+ return _load_dataset_split(
155
+ path,
156
+ validate_split_name(split),
157
+ strict=strict,
158
+ save_index=save_index,
159
+ force_reindex=force_reindex,
160
+ metadata_scope=normalized_scope,
161
+ )
162
+ return index_dataset_from_path_fn(
163
+ path,
164
+ strict=strict,
165
+ save_index=save_index,
166
+ force_reindex=force_reindex,
167
+ metadata_scope=normalized_scope,
168
+ )
169
+ except FileNotFoundError:
170
+ if _has_scoped_metadata(dataset_path, normalized_scope):
171
+ raise
124
172
 
125
173
  if split is None:
126
174
  return index_dataset_from_path_fn(
@@ -131,19 +179,18 @@ def load_index_output(
131
179
  )
132
180
 
133
181
  normalized_split = validate_split_name(split)
134
- if _load_dataset_split is not None:
135
- try:
136
- return _load_dataset_split(
137
- path,
138
- normalized_split,
139
- strict=strict,
140
- save_index=save_index,
141
- force_reindex=force_reindex,
142
- )
143
- except FileNotFoundError:
144
- # Fallback for mocked tests or callers that provide synthetic index
145
- # loaders without a full ds-crawler config on disk.
146
- pass
182
+ try:
183
+ return _load_dataset_split(
184
+ path,
185
+ normalized_split,
186
+ strict=strict,
187
+ save_index=save_index,
188
+ force_reindex=force_reindex,
189
+ )
190
+ except FileNotFoundError:
191
+ # Fallback for mocked tests or callers that provide synthetic index
192
+ # loaders without a full ds-crawler config on disk.
193
+ pass
147
194
 
148
195
  base_output = index_dataset_from_path_fn(
149
196
  path,
@@ -12,6 +12,7 @@ from ._ds_crawler_utils import (
12
12
  extract_ds_crawler_properties,
13
13
  first_non_empty,
14
14
  first_non_empty_list,
15
+ load_dataset_config_for_scope,
15
16
  )
16
17
 
17
18
  if TYPE_CHECKING:
@@ -66,6 +67,8 @@ def _build_runlog_entry(
66
67
  entry: dict[str, Any] = {"path": modality.path}
67
68
  if modality.split is not None:
68
69
  entry["split"] = modality.split
70
+ if modality.metadata_scope is not None:
71
+ entry["metadata_scope"] = modality.metadata_scope
69
72
  if used_as is not None:
70
73
  entry["used_as"] = used_as
71
74
  if slot is not None:
@@ -125,22 +128,26 @@ def _resolve_euler_loading_property(
125
128
  def _get_ds_crawler_descriptor(
126
129
  *,
127
130
  path: str,
131
+ metadata_scope: str | None = None,
128
132
  index_output: Mapping[str, Any] | None,
129
133
  cache: dict[str, dict[str, Any]],
130
134
  load_dataset_config_fn: Callable[[dict[str, Any]], Any],
131
135
  ) -> dict[str, Any]:
132
- if path not in cache:
133
- cache[path] = _read_ds_crawler_descriptor(
136
+ cache_key = f"{path}::{metadata_scope or ''}"
137
+ if cache_key not in cache:
138
+ cache[cache_key] = _read_ds_crawler_descriptor(
134
139
  path=path,
140
+ metadata_scope=metadata_scope,
135
141
  index_output=index_output,
136
142
  load_dataset_config_fn=load_dataset_config_fn,
137
143
  )
138
- return cache[path]
144
+ return cache[cache_key]
139
145
 
140
146
 
141
147
  def _read_ds_crawler_descriptor(
142
148
  *,
143
149
  path: str,
150
+ metadata_scope: str | None,
144
151
  index_output: Mapping[str, Any] | None,
145
152
  load_dataset_config_fn: Callable[[dict[str, Any]], Any],
146
153
  ) -> dict[str, Any]:
@@ -164,7 +171,11 @@ def _read_ds_crawler_descriptor(
164
171
  descriptor["hierarchy_regex"] = hierarchy_regex
165
172
 
166
173
  try:
167
- cfg = load_dataset_config_fn({"path": path})
174
+ cfg = load_dataset_config_for_scope(
175
+ path,
176
+ metadata_scope=metadata_scope,
177
+ load_dataset_config_fn=load_dataset_config_fn,
178
+ )
168
179
  except Exception:
169
180
  return descriptor
170
181
 
@@ -30,7 +30,11 @@ try:
30
30
  except ImportError: # pragma: no cover - compatibility with older ds-crawler
31
31
  _ds_crawler_get_layout_addon = None
32
32
 
33
- from ._ds_crawler_utils import load_index_output, parse_path_with_split
33
+ from ._ds_crawler_utils import (
34
+ load_index_output,
35
+ parse_path_with_split,
36
+ validate_metadata_scope,
37
+ )
34
38
  from ._metadata import _build_runlog_entry, _get_ds_crawler_descriptor
35
39
  from ._resolution import (
36
40
  _resolve_loader,
@@ -328,6 +332,14 @@ class Modality:
328
332
  split: Optional inline split name. When set, euler-loading reads the
329
333
  full ds-crawler metadata from the modality root and overlays the
330
334
  dataset payload from ``.ds_crawler/split_<name>.json``.
335
+ metadata_scope: Optional subdirectory below ``.ds_crawler`` to use for
336
+ this modality's ds-crawler artifacts. This allows multiple
337
+ logical modalities to share one physical directory or zip while
338
+ keeping separate ``dataset-head.json``, ``ds-crawler.json``,
339
+ ``index.json``, and split files under
340
+ ``.ds_crawler/<metadata_scope>/``. When scoped artifacts are
341
+ absent, loading falls back to the legacy root-level
342
+ ``.ds_crawler`` layout.
331
343
  cache: Optional opt-in/out for in-memory caching of loaded values.
332
344
  Only meaningful for hierarchical modalities; regular
333
345
  modalities load once per sample and never cache.
@@ -359,6 +371,7 @@ class Modality:
359
371
  hierarchy_scope: str | None = None
360
372
  applies_to: list[str] | None = None
361
373
  split: str | None = None
374
+ metadata_scope: str | None = None
362
375
  cache: bool | None = None
363
376
  collapse_single: bool = False
364
377
  metadata: dict[str, Any] = field(default_factory=dict)
@@ -374,6 +387,12 @@ class Modality:
374
387
  )
375
388
  object.__setattr__(self, "path", parsed_path)
376
389
  object.__setattr__(self, "split", path_split)
390
+ if self.metadata_scope is not None:
391
+ object.__setattr__(
392
+ self,
393
+ "metadata_scope",
394
+ validate_metadata_scope(self.metadata_scope),
395
+ )
377
396
 
378
397
 
379
398
  class MultiModalDataset(_BaseDataset):
@@ -433,6 +452,8 @@ class MultiModalDataset(_BaseDataset):
433
452
  entry: dict[str, str | None] = {"path": path, "origin_path": origin}
434
453
  if mod.split is not None:
435
454
  entry["split"] = mod.split
455
+ if mod.metadata_scope is not None:
456
+ entry["metadata_scope"] = mod.metadata_scope
436
457
  res[name] = entry
437
458
  return res
438
459
 
@@ -450,6 +471,8 @@ class MultiModalDataset(_BaseDataset):
450
471
  entry: dict[str, str | None] = {"path": path, "origin_path": origin}
451
472
  if mod.split is not None:
452
473
  entry["split"] = mod.split
474
+ if mod.metadata_scope is not None:
475
+ entry["metadata_scope"] = mod.metadata_scope
453
476
  res[name] = entry
454
477
  return res
455
478
 
@@ -496,6 +519,7 @@ class MultiModalDataset(_BaseDataset):
496
519
  for name, modality in self._modalities.items():
497
520
  descriptor = _get_ds_crawler_descriptor(
498
521
  path=modality.path,
522
+ metadata_scope=modality.metadata_scope,
499
523
  index_output=self._index_outputs.get(name),
500
524
  cache=descriptor_cache,
501
525
  load_dataset_config_fn=load_dataset_config,
@@ -513,6 +537,7 @@ class MultiModalDataset(_BaseDataset):
513
537
  for name, modality in self._hierarchical_modalities.items():
514
538
  descriptor = _get_ds_crawler_descriptor(
515
539
  path=modality.path,
540
+ metadata_scope=modality.metadata_scope,
516
541
  index_output=self._hierarchical_index_outputs.get(name),
517
542
  cache=descriptor_cache,
518
543
  load_dataset_config_fn=load_dataset_config,
@@ -562,6 +587,7 @@ class MultiModalDataset(_BaseDataset):
562
587
  index = load_index_output(
563
588
  modality.path,
564
589
  split=modality.split,
590
+ metadata_scope=modality.metadata_scope,
565
591
  index_dataset_from_path_fn=index_dataset_from_path,
566
592
  )
567
593
  indexed[name] = index
@@ -653,6 +679,7 @@ class MultiModalDataset(_BaseDataset):
653
679
  index = load_index_output(
654
680
  modality.path,
655
681
  split=modality.split,
682
+ metadata_scope=modality.metadata_scope,
656
683
  index_dataset_from_path_fn=index_dataset_from_path,
657
684
  )
658
685
  self._index_outputs[name] = index
@@ -688,6 +715,7 @@ class MultiModalDataset(_BaseDataset):
688
715
  index = load_index_output(
689
716
  modality.path,
690
717
  split=modality.split,
718
+ metadata_scope=modality.metadata_scope,
691
719
  index_dataset_from_path_fn=index_dataset_from_path,
692
720
  )
693
721
  self._hierarchical_index_outputs[name] = index
@@ -16,11 +16,17 @@ Return types
16
16
  containing COCO-style panoptic segment IDs decoded from RGB.
17
17
  - **lidar_point_cloud** / **sparse_depth** -- ``np.ndarray`` of shape
18
18
  ``(N, 6)`` float64 with columns ``x, y, z, intensity, ring, timestamp``.
19
+ - **read_intrinsics** -- ``np.ndarray`` of shape ``(3, 3)`` float32 parsed
20
+ from ``calib.json``.
21
+ - **read_extrinsics** -- ``np.ndarray`` of shape ``(4, 4)`` float32 parsed
22
+ from ``calib.json``.
19
23
  """
20
24
 
21
25
  from __future__ import annotations
22
26
 
27
+ import json
23
28
  import os
29
+ from collections.abc import Mapping
24
30
  from typing import Any, BinaryIO, Union
25
31
 
26
32
  import numpy as np
@@ -78,6 +84,128 @@ def _load_lidar_array(path: Union[str, BinaryIO]) -> np.ndarray:
78
84
  return arr.reshape((-1, 6))
79
85
 
80
86
 
87
+ def _load_json(path: Union[str, BinaryIO]) -> dict[str, Any]:
88
+ """Load JSON from a file path or an in-memory buffer."""
89
+ if isinstance(path, (str, os.PathLike)):
90
+ with open(path) as f:
91
+ data = json.load(f)
92
+ else:
93
+ data = json.load(path)
94
+ if not isinstance(data, dict):
95
+ name = getattr(path, "name", path)
96
+ raise ValueError(f"MUSES calibration file {name!r} must contain a JSON object.")
97
+ return data
98
+
99
+
100
+ def _select_string(
101
+ *,
102
+ meta: Mapping[str, Any] | None,
103
+ attributes: Mapping[str, Any] | None,
104
+ keys: tuple[str, ...],
105
+ ) -> str | None:
106
+ for source in (attributes, meta):
107
+ if not isinstance(source, Mapping):
108
+ continue
109
+ for key in keys:
110
+ value = source.get(key)
111
+ if isinstance(value, str) and value:
112
+ return value
113
+ return None
114
+
115
+
116
+ def _available_keys(section: Mapping[str, Any]) -> str:
117
+ return ", ".join(sorted(str(key) for key in section.keys()))
118
+
119
+
120
+ def _read_intrinsics_array(
121
+ path: Union[str, BinaryIO],
122
+ meta: Mapping[str, Any] | None,
123
+ attributes: Mapping[str, Any] | None,
124
+ ) -> np.ndarray:
125
+ data = _load_json(path)
126
+ intrinsics = data.get("intrinsics")
127
+ if not isinstance(intrinsics, Mapping):
128
+ raise ValueError("MUSES calibration JSON must contain an 'intrinsics' object.")
129
+
130
+ sensor = _select_string(
131
+ meta=meta,
132
+ attributes=attributes,
133
+ keys=("sensor", "camera", "intrinsics_sensor", "target_sensor"),
134
+ ) or "rgb"
135
+ entry = intrinsics.get(sensor)
136
+ if not isinstance(entry, Mapping) or "K" not in entry:
137
+ raise KeyError(
138
+ f"MUSES calibration has no intrinsics for sensor {sensor!r}. "
139
+ f"Available sensors: {_available_keys(intrinsics)}"
140
+ )
141
+
142
+ K = np.asarray(entry["K"], dtype=np.float32)
143
+ if K.shape != (3, 3):
144
+ raise ValueError(
145
+ f"MUSES intrinsics for sensor {sensor!r} must have shape (3, 3), "
146
+ f"got {K.shape}."
147
+ )
148
+ return K
149
+
150
+
151
+ def _select_extrinsics_key(
152
+ *,
153
+ meta: Mapping[str, Any] | None,
154
+ attributes: Mapping[str, Any] | None,
155
+ ) -> str:
156
+ explicit = _select_string(
157
+ meta=meta,
158
+ attributes=attributes,
159
+ keys=("transform", "extrinsics_key", "extrinsic", "key"),
160
+ )
161
+ if explicit is not None:
162
+ return explicit
163
+
164
+ source_sensor = _select_string(
165
+ meta=meta,
166
+ attributes=attributes,
167
+ keys=("source_sensor", "from_sensor", "source"),
168
+ )
169
+ target_sensor = _select_string(
170
+ meta=meta,
171
+ attributes=attributes,
172
+ keys=("target_sensor", "to_sensor", "target", "camera"),
173
+ )
174
+ if source_sensor is not None and target_sensor is not None:
175
+ return f"{source_sensor}2{target_sensor}"
176
+
177
+ return "lidar2rgb"
178
+
179
+
180
+ def _read_extrinsics_array(
181
+ path: Union[str, BinaryIO],
182
+ meta: Mapping[str, Any] | None,
183
+ attributes: Mapping[str, Any] | None,
184
+ ) -> np.ndarray:
185
+ data = _load_json(path)
186
+ extrinsics = data.get("extrinsics")
187
+ if not isinstance(extrinsics, Mapping):
188
+ raise ValueError("MUSES calibration JSON must contain an 'extrinsics' object.")
189
+
190
+ transform = _select_extrinsics_key(meta=meta, attributes=attributes)
191
+ raw = extrinsics.get(transform)
192
+ if raw is None:
193
+ raise KeyError(
194
+ f"MUSES calibration has no extrinsics transform {transform!r}. "
195
+ f"Available transforms: {_available_keys(extrinsics)}"
196
+ )
197
+
198
+ T = np.asarray(raw, dtype=np.float32)
199
+ if T.shape == (3, 4):
200
+ T = np.vstack([T, np.array([[0.0, 0.0, 0.0, 1.0]], dtype=np.float32)])
201
+ if T.shape != (4, 4):
202
+ raise ValueError(
203
+ f"MUSES extrinsics transform {transform!r} must have shape "
204
+ f"(4, 4) or (3, 4), got {T.shape}."
205
+ )
206
+ return T
207
+
208
+
81
209
  @modality_meta(
82
210
  modality_type="rgb",
83
211
  dtype="float32",
@@ -178,3 +306,43 @@ def point_cloud(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None,
178
306
  def sparse_depth(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None, *, attributes: dict[str, Any] | None = None) -> np.ndarray:
179
307
  """Load MUSES lidar points for sparse-depth style workflows."""
180
308
  return _load_lidar_array(path)
309
+
310
+
311
+ @modality_meta(
312
+ modality_type="intrinsics",
313
+ dtype="float32",
314
+ hierarchical=True,
315
+ shape="3x3",
316
+ file_formats=[".json"],
317
+ meta={"default_sensor": "rgb"},
318
+ )
319
+ def read_intrinsics(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None, *, attributes: dict[str, Any] | None = None) -> np.ndarray:
320
+ """Load a MUSES camera intrinsics matrix from ``calib.json``.
321
+
322
+ The default sensor is ``"rgb"``. Override it with ``meta`` or per-file
323
+ ``attributes`` using ``sensor``, ``camera``, ``intrinsics_sensor``, or
324
+ ``target_sensor``.
325
+ """
326
+ return _read_intrinsics_array(path, meta, attributes)
327
+
328
+
329
+ @modality_meta(
330
+ modality_type="camera_extrinsics",
331
+ dtype="float32",
332
+ hierarchical=True,
333
+ shape="4x4",
334
+ file_formats=[".json"],
335
+ meta={
336
+ "default_transform": "lidar2rgb",
337
+ "transform_direction": "source_sensor_to_target_sensor",
338
+ },
339
+ )
340
+ def read_extrinsics(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None, *, attributes: dict[str, Any] | None = None) -> np.ndarray:
341
+ """Load a MUSES camera extrinsics matrix from ``calib.json``.
342
+
343
+ The default transform is ``"lidar2rgb"``, suitable for projecting MUSES
344
+ lidar/sparse-depth points into the RGB camera. Override it with ``meta`` or
345
+ per-file ``attributes`` using ``transform`` / ``extrinsics_key`` or
346
+ ``source_sensor`` + ``target_sensor``.
347
+ """
348
+ return _read_extrinsics_array(path, meta, attributes)