euler-loading 2.0.0__tar.gz → 2.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. {euler_loading-2.0.0 → euler_loading-2.2.0}/PKG-INFO +1 -1
  2. {euler_loading-2.0.0 → euler_loading-2.2.0}/README.md +38 -0
  3. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/__init__.py +16 -0
  4. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/_ds_crawler_utils.py +20 -9
  5. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/_resolution.py +17 -10
  6. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/_writing.py +122 -4
  7. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/dataset.py +48 -25
  8. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/cpu/generic.py +71 -0
  9. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/gpu/generic.py +73 -0
  10. euler_loading-2.2.0/euler_loading/preprocessing.py +804 -0
  11. {euler_loading-2.0.0 → euler_loading-2.2.0}/package-lock.json +1 -1
  12. {euler_loading-2.0.0 → euler_loading-2.2.0}/pyproject.toml +1 -1
  13. euler_loading-2.2.0/tests/test_preprocessing.py +185 -0
  14. {euler_loading-2.0.0 → euler_loading-2.2.0}/.github/workflows/workflow.yml +0 -0
  15. {euler_loading-2.0.0 → euler_loading-2.2.0}/.gitignore +0 -0
  16. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/_dataset_contract.py +0 -0
  17. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/_metadata.py +0 -0
  18. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/indexing.py +0 -0
  19. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/__init__.py +0 -0
  20. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/_annotations.py +0 -0
  21. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/_writer_utils.py +0 -0
  22. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/contracts.py +0 -0
  23. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/cpu/__init__.py +0 -0
  24. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/cpu/generic_dense_depth.py +0 -0
  25. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/cpu/real_drive_sim.py +0 -0
  26. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/cpu/vkitti2.py +0 -0
  27. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/generate/__init__.py +0 -0
  28. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/generate/__main__.py +0 -0
  29. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/generate/loaders.json +0 -0
  30. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/generic.py +0 -0
  31. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/gpu/__init__.py +0 -0
  32. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/gpu/generic_dense_depth.py +0 -0
  33. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/gpu/real_drive_sim.py +0 -0
  34. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/gpu/vkitti2.py +0 -0
  35. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/real_drive_sim.py +0 -0
  36. {euler_loading-2.0.0 → euler_loading-2.2.0}/euler_loading/loaders/vkitti2.py +0 -0
  37. {euler_loading-2.0.0 → euler_loading-2.2.0}/example.py +0 -0
  38. {euler_loading-2.0.0 → euler_loading-2.2.0}/sample_rds.py +0 -0
  39. {euler_loading-2.0.0 → euler_loading-2.2.0}/tests/__init__.py +0 -0
  40. {euler_loading-2.0.0 → euler_loading-2.2.0}/tests/conftest.py +0 -0
  41. {euler_loading-2.0.0 → euler_loading-2.2.0}/tests/example_rds_calib.json +0 -0
  42. {euler_loading-2.0.0 → euler_loading-2.2.0}/tests/test_dataset.py +0 -0
  43. {euler_loading-2.0.0 → euler_loading-2.2.0}/tests/test_indexing.py +0 -0
  44. {euler_loading-2.0.0 → euler_loading-2.2.0}/tests/test_loaders.py +0 -0
  45. {euler_loading-2.0.0 → euler_loading-2.2.0}/tests/test_real_dataset.py +0 -0
  46. {euler_loading-2.0.0 → euler_loading-2.2.0}/tests/test_writing.py +0 -0
  47. {euler_loading-2.0.0 → euler_loading-2.2.0}/vkitti_cpu_example_output.json +0 -0
  48. {euler_loading-2.0.0 → euler_loading-2.2.0}/vkitti_gpu_example_output.json +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: euler-loading
3
- Version: 2.0.0
3
+ Version: 2.2.0
4
4
  Summary: Multi-modal PyTorch dataloader using ds-crawler indices
5
5
  Requires-Python: >=3.9
6
6
  Requires-Dist: ds-crawler
@@ -218,6 +218,44 @@ def mask_sky_in_depth(sample: dict) -> dict:
218
218
  return sample
219
219
  ```
220
220
 
221
+ ### Built-in spatial preprocessing
222
+
223
+ `euler_loading.SamplePreprocessor` applies shared spatial ops such as resize and crop
224
+ consistently across sample fields, including calibration-sensitive fields such as
225
+ camera intrinsics and ray maps.
226
+
227
+ ```python
228
+ from euler_loading import FieldSpec, SamplePreprocessor
229
+
230
+ preprocessor = SamplePreprocessor.from_config(
231
+ {
232
+ "resize": [384, 768],
233
+ "crop": {"size": [320, 640], "anchor": "center"},
234
+ "fields": {
235
+ "rgb": {"kind": "image"},
236
+ "depth": {"kind": "depth"},
237
+ "valid_mask": {"kind": "mask"},
238
+ "intrinsics": {"kind": "intrinsics", "reduce": "first"},
239
+ "ray_map": {"kind": "ray_map"},
240
+ },
241
+ }
242
+ )
243
+
244
+ dataset = MultiModalDataset(
245
+ modalities={...},
246
+ hierarchical_modalities={...},
247
+ transforms=[preprocessor],
248
+ )
249
+ ```
250
+
251
+ - `kind="image"` and `kind="depth"` default to bilinear interpolation.
252
+ - `kind="mask"` defaults to nearest-neighbour interpolation and preserves mask dtype.
253
+ - `kind="ray_map"` bilinearly resizes and renormalizes vectors to unit length.
254
+ - `kind="intrinsics"` rescales the camera matrix for resize and shifts the principal point for crop.
255
+ - `reduce="first"` is useful for hierarchical modalities such as one intrinsics file per scene/camera.
256
+
257
+ If you already set `modality_type` on your `Modality(...)` definitions, the preprocessor can bind to the dataset and reuse those hints automatically.
258
+
221
259
  ## Zip archive support
222
260
 
223
261
  Modality paths can point to `.zip` files instead of directories. euler-loading detects zip paths automatically and reads files directly from the archive without extraction:
@@ -6,15 +6,31 @@ from ._writing import create_dataset_writer_from_index
6
6
  from .dataset import Modality, MultiModalDataset
7
7
  from .indexing import FileRecord
8
8
  from .loaders.contracts import DenseDepthCodec, DenseDepthLoader, DenseDepthWriter
9
+ from .preprocessing import (
10
+ Crop,
11
+ FieldSpec,
12
+ Resize,
13
+ SamplePreprocessor,
14
+ crop_intrinsics,
15
+ infer_field_spec,
16
+ resize_intrinsics,
17
+ )
9
18
 
10
19
  __all__ = [
11
20
  "DenseDepthCodec",
12
21
  "DenseDepthLoader",
13
22
  "DenseDepthWriter",
14
23
  "FileRecord",
24
+ "Crop",
25
+ "FieldSpec",
15
26
  "Modality",
16
27
  "MultiModalDataset",
28
+ "Resize",
29
+ "SamplePreprocessor",
17
30
  "create_dataset_writer_from_index",
31
+ "crop_intrinsics",
32
+ "infer_field_spec",
18
33
  "resolve_loader_module",
19
34
  "resolve_writer_module",
35
+ "resize_intrinsics",
20
36
  ]
@@ -49,7 +49,13 @@ def first_non_empty_list(*candidates: list[str] | None) -> list[str]:
49
49
 
50
50
 
51
51
  def extract_ds_crawler_properties(index_output: Mapping[str, Any]) -> dict[str, Any]:
52
- return get_dataset_contract(dict(index_output)).to_properties_dict()
52
+ try:
53
+ return get_dataset_contract(dict(index_output)).to_properties_dict()
54
+ except Exception:
55
+ properties = index_output.get("properties")
56
+ if isinstance(properties, Mapping):
57
+ return dict(properties)
58
+ return {}
53
59
 
54
60
 
55
61
  _SPLIT_NAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
@@ -126,13 +132,18 @@ def load_index_output(
126
132
 
127
133
  normalized_split = validate_split_name(split)
128
134
  if _load_dataset_split is not None:
129
- return _load_dataset_split(
130
- path,
131
- normalized_split,
132
- strict=strict,
133
- save_index=save_index,
134
- force_reindex=force_reindex,
135
- )
135
+ try:
136
+ return _load_dataset_split(
137
+ path,
138
+ normalized_split,
139
+ strict=strict,
140
+ save_index=save_index,
141
+ force_reindex=force_reindex,
142
+ )
143
+ except FileNotFoundError:
144
+ # Fallback for mocked tests or callers that provide synthetic index
145
+ # loaders without a full ds-crawler config on disk.
146
+ pass
136
147
 
137
148
  base_output = index_dataset_from_path_fn(
138
149
  path,
@@ -149,5 +160,5 @@ def load_index_output(
149
160
 
150
161
  result = dict(base_output)
151
162
  result["index"] = split_dataset
152
- result.pop("dataset", None)
163
+ result["dataset"] = split_dataset
153
164
  return result
@@ -25,6 +25,21 @@ _LOADER_MODULES: dict[str, str] = {
25
25
  }
26
26
 
27
27
 
28
+ def _get_euler_loading_meta(index: Mapping[str, Any]) -> Mapping[str, Any] | None:
29
+ try:
30
+ contract = get_dataset_contract(dict(index))
31
+ euler_loading_meta = contract.get_addon("euler_loading")
32
+ except Exception:
33
+ euler_loading_meta = None
34
+ if isinstance(euler_loading_meta, Mapping):
35
+ return euler_loading_meta
36
+
37
+ raw_meta = index.get("euler_loading")
38
+ if isinstance(raw_meta, Mapping):
39
+ return raw_meta
40
+ return None
41
+
42
+
28
43
  def resolve_loader_module(name: str) -> ModuleType:
29
44
  """Import and return the GPU loader module for *name*.
30
45
 
@@ -63,11 +78,7 @@ def _resolve_loader(
63
78
  if modality.loader is not None:
64
79
  return modality.loader
65
80
 
66
- try:
67
- contract = get_dataset_contract(index)
68
- euler_loading_meta = contract.get_addon("euler_loading")
69
- except Exception:
70
- euler_loading_meta = None
81
+ euler_loading_meta = _get_euler_loading_meta(index)
71
82
  if not isinstance(euler_loading_meta, Mapping):
72
83
  raise ValueError(
73
84
  f"Modality {modality_name!r}: no explicit loader provided and the "
@@ -132,11 +143,7 @@ def _resolve_writer(
132
143
  if modality.writer is not None:
133
144
  return modality.writer
134
145
 
135
- try:
136
- contract = get_dataset_contract(index)
137
- euler_loading_meta = contract.get_addon("euler_loading")
138
- except Exception:
139
- euler_loading_meta = None
146
+ euler_loading_meta = _get_euler_loading_meta(index)
140
147
  if not isinstance(euler_loading_meta, Mapping):
141
148
  return None
142
149
 
@@ -1,8 +1,10 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import json
3
4
  import logging
4
5
  import os
5
6
  import tempfile
7
+ import zipfile
6
8
  from collections.abc import Mapping
7
9
  from pathlib import Path
8
10
  from typing import Any, Callable
@@ -18,10 +20,93 @@ from .loaders._writer_utils import supports_stream_target
18
20
 
19
21
  logger = logging.getLogger(__name__)
20
22
 
23
+ _DEFAULT_WRITER_INDEX_FILENAME = "output.json"
24
+
21
25
 
22
26
  OutputDestination = str | os.PathLike[str] | DatasetWriter | ZipDatasetWriter
23
27
 
24
28
 
29
+ def _wrap_writer_save_index(
30
+ writer: DatasetWriter | ZipDatasetWriter,
31
+ ) -> DatasetWriter | ZipDatasetWriter:
32
+ original_save_index = writer.save_index
33
+
34
+ def _build_legacy_fields() -> dict[str, Any]:
35
+ output = writer.build_output()
36
+ result: dict[str, Any] = {}
37
+
38
+ head = output.get("head")
39
+ if isinstance(head, Mapping):
40
+ dataset = head.get("dataset")
41
+ modality = head.get("modality")
42
+ addons = head.get("addons")
43
+
44
+ if isinstance(dataset, Mapping):
45
+ name = as_non_empty_str(dataset.get("name"))
46
+ if name is not None:
47
+ result["name"] = name
48
+
49
+ if isinstance(modality, Mapping):
50
+ modality_key = as_non_empty_str(modality.get("key"))
51
+ if modality_key is not None:
52
+ result["type"] = modality_key
53
+ meta = modality.get("meta")
54
+ if isinstance(meta, Mapping):
55
+ result["meta"] = dict(meta)
56
+
57
+ if isinstance(addons, Mapping):
58
+ for addon_name in ("euler_train", "euler_loading"):
59
+ addon = addons.get(addon_name)
60
+ if isinstance(addon, Mapping):
61
+ result[addon_name] = dict(addon)
62
+
63
+ index_tree = output.get("index")
64
+ if isinstance(index_tree, Mapping):
65
+ result["dataset"] = dict(index_tree)
66
+
67
+ return result
68
+
69
+ def save_index(filename: str = _DEFAULT_WRITER_INDEX_FILENAME):
70
+ legacy_fields = _build_legacy_fields()
71
+ path = original_save_index(filename=filename)
72
+
73
+ if isinstance(writer, ZipDatasetWriter):
74
+ entry_name = f".ds_crawler/{filename}"
75
+ archive_path = Path(writer.root)
76
+ with zipfile.ZipFile(archive_path, "r") as source_zip:
77
+ output_index = json.loads(source_zip.read(entry_name).decode("utf-8"))
78
+ output_index.update(legacy_fields)
79
+
80
+ replacement_name = archive_path.with_suffix(archive_path.suffix + ".tmp")
81
+ with zipfile.ZipFile(replacement_name, "w") as target_zip:
82
+ replaced = False
83
+ for info in source_zip.infolist():
84
+ data = source_zip.read(info.filename)
85
+ if info.filename == entry_name and not replaced:
86
+ data = json.dumps(output_index, indent=2).encode("utf-8")
87
+ replaced = True
88
+ target_zip.writestr(info, data)
89
+
90
+ if not replaced:
91
+ target_zip.writestr(
92
+ entry_name,
93
+ json.dumps(output_index, indent=2),
94
+ compress_type=zipfile.ZIP_DEFLATED,
95
+ )
96
+
97
+ replacement_name.replace(archive_path)
98
+ return path
99
+
100
+ output_path = Path(path)
101
+ output_index = json.loads(output_path.read_text(encoding="utf-8"))
102
+ output_index.update(legacy_fields)
103
+ output_path.write_text(json.dumps(output_index, indent=2), encoding="utf-8")
104
+ return path
105
+
106
+ setattr(writer, "save_index", save_index)
107
+ return writer
108
+
109
+
25
110
  def create_dataset_writer_from_index(
26
111
  *,
27
112
  index_output: Mapping[str, Any],
@@ -29,8 +114,6 @@ def create_dataset_writer_from_index(
29
114
  zip: bool = False,
30
115
  ) -> DatasetWriter | ZipDatasetWriter:
31
116
  """Create a ds-crawler writer that mirrors an existing index's metadata."""
32
- contract = get_dataset_contract(dict(index_output))
33
-
34
117
  indexing = index_output.get("indexing")
35
118
  hierarchy = indexing.get("hierarchy") if isinstance(indexing, Mapping) else None
36
119
  id_cfg = indexing.get("id") if isinstance(indexing, Mapping) else None
@@ -39,11 +122,46 @@ def create_dataset_writer_from_index(
39
122
  as_non_empty_str(id_cfg.get("join_char")) if isinstance(id_cfg, Mapping) else None,
40
123
  )
41
124
  writer_cls = ZipDatasetWriter if zip else DatasetWriter
42
- return writer_cls(
125
+ try:
126
+ contract = get_dataset_contract(dict(index_output))
127
+ except Exception:
128
+ contract = None
129
+
130
+ if contract is not None:
131
+ writer = writer_cls(
132
+ root,
133
+ head=contract.to_mapping(),
134
+ separator=separator,
135
+ )
136
+ return _wrap_writer_save_index(writer)
137
+
138
+ name = as_non_empty_str(index_output.get("name"))
139
+ type_name = as_non_empty_str(index_output.get("type"))
140
+ if name is None or type_name is None:
141
+ raise ValueError(
142
+ "Cannot create a dataset writer without ds-crawler head metadata. "
143
+ "Expected either a dataset contract or top-level 'name' and 'type'."
144
+ )
145
+
146
+ euler_train = index_output.get("euler_train")
147
+ euler_loading = index_output.get("euler_loading")
148
+ meta = index_output.get("meta")
149
+
150
+ properties: dict[str, Any] = {}
151
+ if isinstance(euler_loading, Mapping):
152
+ properties["euler_loading"] = dict(euler_loading)
153
+ if isinstance(meta, Mapping):
154
+ properties["meta"] = dict(meta)
155
+
156
+ writer = writer_cls(
43
157
  root,
44
- head=contract.to_mapping(),
158
+ name=name,
159
+ type=type_name,
160
+ euler_train=dict(euler_train) if isinstance(euler_train, Mapping) else None,
45
161
  separator=separator,
162
+ **properties,
46
163
  )
164
+ return _wrap_writer_save_index(writer)
47
165
 
48
166
 
49
167
  def _resolve_output_destination(
@@ -59,6 +59,39 @@ __all__ = [
59
59
  ]
60
60
 
61
61
 
62
+ def _get_index_tree(index_output: Mapping[str, Any]) -> dict[str, Any]:
63
+ """Return the dataset tree from a ds-crawler output payload."""
64
+ index_tree = index_output.get("index")
65
+ if isinstance(index_tree, Mapping):
66
+ dataset_tree = index_tree.get("dataset")
67
+ if isinstance(dataset_tree, Mapping):
68
+ return dict(dataset_tree)
69
+ return dict(index_tree)
70
+
71
+ dataset_tree = index_output.get("dataset")
72
+ if isinstance(dataset_tree, Mapping):
73
+ return dict(dataset_tree)
74
+
75
+ raise KeyError(
76
+ "ds-crawler output must contain either an 'index' tree or a 'dataset' tree."
77
+ )
78
+
79
+
80
+ def _get_index_meta(index_output: Mapping[str, Any]) -> dict[str, Any] | None:
81
+ """Return modality meta from a ds-crawler output payload when available."""
82
+ try:
83
+ meta = get_dataset_contract(dict(index_output)).meta
84
+ except Exception:
85
+ meta = None
86
+ if isinstance(meta, Mapping):
87
+ return dict(meta)
88
+
89
+ raw_meta = index_output.get("meta")
90
+ if isinstance(raw_meta, Mapping):
91
+ return dict(raw_meta)
92
+ return None
93
+
94
+
62
95
  #TODO: might want to add slots=True in a 3.10+ only codebase
63
96
  @dataclass(frozen=True)
64
97
  class Modality:
@@ -320,7 +353,7 @@ class MultiModalDataset(_BaseDataset):
320
353
  self._resolved_writers[name] = _resolve_writer(
321
354
  modality_name=name, modality=modality, index=index,
322
355
  )
323
- records = collect_files(index["index"])
356
+ records = collect_files(_get_index_tree(index))
324
357
  lookup = {
325
358
  "/".join(rec.hierarchy_path + (rec.file_entry["id"],)): rec
326
359
  for rec in records
@@ -352,7 +385,7 @@ class MultiModalDataset(_BaseDataset):
352
385
  self._resolved_writers[name] = _resolve_writer(
353
386
  modality_name=name, modality=modality, index=index,
354
387
  )
355
- files_by_level = collect_hierarchical_files(index["index"])
388
+ files_by_level = collect_hierarchical_files(_get_index_tree(index))
356
389
  self._hierarchical_lookups[name] = files_by_level
357
390
  total_files = sum(len(f) for f in files_by_level.values())
358
391
  logger.info(
@@ -406,13 +439,17 @@ class MultiModalDataset(_BaseDataset):
406
439
  # -- Caches ----------------------------------------------------------
407
440
  self._hierarchical_cache: dict[str, Any] = {}
408
441
 
442
+ # -- Optional transform binding -------------------------------------
443
+ for transform in self._transforms:
444
+ bind = getattr(transform, "bind_to_dataset", None)
445
+ if callable(bind):
446
+ bind(self)
447
+
409
448
  def get_modality_metadata(self, modality_name: str) -> dict[str, Any]:
410
449
  """Return the metadata dict for a given modality name."""
411
450
  index = self._index_outputs.get(modality_name, {})
412
- try:
413
- return get_dataset_contract(index).meta or {}
414
- except Exception:
415
- return {}
451
+ meta = _get_index_meta(index)
452
+ return meta or {}
416
453
 
417
454
  def get_modality_index(self, modality_name: str) -> dict[str, Any]:
418
455
  """Return the cached ds-crawler index for a modality."""
@@ -510,12 +547,7 @@ class MultiModalDataset(_BaseDataset):
510
547
  output_root=output_root, modality_name=modality_name
511
548
  )
512
549
  record = self._lookups[modality_name][sample_id]
513
- try:
514
- modality_meta = get_dataset_contract(
515
- self._index_outputs[modality_name]
516
- ).meta
517
- except Exception:
518
- modality_meta = None
550
+ modality_meta = _get_index_meta(self._index_outputs[modality_name])
519
551
  basename = Path(record.file_entry["path"]).name
520
552
  relative_path = record.file_entry["path"]
521
553
  full_id = _build_writer_full_id(
@@ -558,7 +590,8 @@ class MultiModalDataset(_BaseDataset):
558
590
  try:
559
591
  name = get_dataset_contract(index).name
560
592
  except Exception:
561
- name = None
593
+ raw_name = index.get("name")
594
+ name = str(raw_name) if raw_name is not None else None
562
595
  if name is None:
563
596
  continue
564
597
  if first_name is None:
@@ -617,12 +650,7 @@ class MultiModalDataset(_BaseDataset):
617
650
 
618
651
  for name, modality in self._modalities.items():
619
652
  record = self._lookups[name][sample_id]
620
- try:
621
- modality_meta = get_dataset_contract(
622
- self._index_outputs[name]
623
- ).meta
624
- except Exception:
625
- modality_meta = None
653
+ modality_meta = _get_index_meta(self._index_outputs[name])
626
654
 
627
655
  if name in self._zip_modalities:
628
656
  file_or_path = self._open_from_zip(
@@ -643,12 +671,7 @@ class MultiModalDataset(_BaseDataset):
643
671
  for name, modality in self._hierarchical_modalities.items():
644
672
  files_by_level = self._hierarchical_lookups[name]
645
673
  matched = match_hierarchical_files(hierarchy_path, files_by_level)
646
- try:
647
- modality_meta = get_dataset_contract(
648
- self._hierarchical_index_outputs[name]
649
- ).meta
650
- except Exception:
651
- modality_meta = None
674
+ modality_meta = _get_index_meta(self._hierarchical_index_outputs[name])
652
675
  loaded: dict[str, Any] = {}
653
676
  for entry in matched:
654
677
  cache_key = f"{modality.path}/{entry['path']}"
@@ -11,6 +11,8 @@ Supported file extensions:
11
11
  Return types
12
12
  ------------
13
13
  - **spherical_map** -- ``np.ndarray`` of shape ``(H, W, C)`` float32.
14
+ - **intrinsics** -- ``np.ndarray`` of shape ``(3, 3)`` float32.
15
+ - **sh_coeffs** -- ``np.ndarray`` of shape ``(N, 3)`` float32.
14
16
 
15
17
  Usage::
16
18
 
@@ -18,6 +20,8 @@ Usage::
18
20
  from euler_loading import Modality
19
21
 
20
22
  Modality("/data/my_dataset/spherical_map", loader=generic.spherical_map)
23
+ Modality("/data/my_dataset/intrinsics", loader=generic.intrinsics)
24
+ Modality("/data/my_dataset/sh_coeffs", loader=generic.sh_coeffs)
21
25
  """
22
26
 
23
27
  from __future__ import annotations
@@ -60,6 +64,22 @@ def _load_numpy(path: Union[str, BinaryIO]) -> np.ndarray:
60
64
  return arr.astype(np.float32)
61
65
 
62
66
 
67
+ def _write_numpy(path: Union[str, BinaryIO], value: Any) -> None:
68
+ """Write an array to ``.npy`` or ``.npz`` based on extension."""
69
+ ensure_parent(path)
70
+ ext = os.path.splitext(_get_name(path))[1].lower()
71
+ arr = to_numpy(value).astype(np.float32)
72
+
73
+ if ext == _NPY_EXTENSION:
74
+ np.save(path, arr)
75
+ return
76
+ if ext == _NPZ_EXTENSION:
77
+ np.savez_compressed(path, data=arr)
78
+ return
79
+
80
+ raise ValueError(f"Unsupported output extension: {ext}")
81
+
82
+
63
83
  # ---------------------------------------------------------------------------
64
84
  # Public loaders
65
85
  # ---------------------------------------------------------------------------
@@ -86,6 +106,45 @@ def spherical_map(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None
86
106
  # ---------------------------------------------------------------------------
87
107
 
88
108
 
109
+ @modality_meta(
110
+ modality_type="intrinsics",
111
+ dtype="float32",
112
+ shape="3x3",
113
+ file_formats=[".npy", ".npz"],
114
+ )
115
+ def intrinsics(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> np.ndarray:
116
+ """Load a camera intrinsics matrix as a ``(3, 3)`` float32 array.
117
+
118
+ The file is expected to contain a ``(3, 3)`` array::
119
+
120
+ [[fx, 0, cx],
121
+ [ 0, fy, cy],
122
+ [ 0, 0, 1]]
123
+ """
124
+ return _load_numpy(path)
125
+
126
+
127
+ @modality_meta(
128
+ modality_type="sh_coeffs",
129
+ dtype="float32",
130
+ shape="NC",
131
+ file_formats=[".npy", ".npz"],
132
+ )
133
+ def sh_coeffs(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> np.ndarray:
134
+ """Load spherical-harmonic coefficients as an ``(N, 3)`` float32 array.
135
+
136
+ *N* is the number of SH basis functions (e.g. 15 for degree-3 SH with
137
+ the constant term removed). Each row is a 3-vector (one per spatial
138
+ dimension).
139
+ """
140
+ return _load_numpy(path)
141
+
142
+
143
+ # ---------------------------------------------------------------------------
144
+ # Writers
145
+ # ---------------------------------------------------------------------------
146
+
147
+
89
148
  @mark_stream_supported
90
149
  def write_spherical_map(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
91
150
  """Write spherical-map data to NumPy formats based on extension.
@@ -107,3 +166,15 @@ def write_spherical_map(path: Union[str, BinaryIO], value: Any, meta: dict[str,
107
166
  return
108
167
 
109
168
  raise ValueError(f"Unsupported spherical-map output extension: {ext}")
169
+
170
+
171
+ @mark_stream_supported
172
+ def write_intrinsics(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
173
+ """Write a ``(3, 3)`` intrinsics matrix to a NumPy file."""
174
+ _write_numpy(path, value)
175
+
176
+
177
+ @mark_stream_supported
178
+ def write_sh_coeffs(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
179
+ """Write ``(N, 3)`` spherical-harmonic coefficients to a NumPy file."""
180
+ _write_numpy(path, value)
@@ -11,6 +11,8 @@ Supported file extensions:
11
11
  Return types
12
12
  ------------
13
13
  - **spherical_map** -- ``torch.FloatTensor`` of shape ``(C, H, W)``.
14
+ - **intrinsics** -- ``torch.FloatTensor`` of shape ``(3, 3)``.
15
+ - **sh_coeffs** -- ``torch.FloatTensor`` of shape ``(N, 3)``.
14
16
 
15
17
  Usage::
16
18
 
@@ -18,6 +20,8 @@ Usage::
18
20
  from euler_loading import Modality
19
21
 
20
22
  Modality("/data/my_dataset/spherical_map", loader=generic.spherical_map)
23
+ Modality("/data/my_dataset/intrinsics", loader=generic.intrinsics)
24
+ Modality("/data/my_dataset/sh_coeffs", loader=generic.sh_coeffs)
21
25
  """
22
26
 
23
27
  from __future__ import annotations
@@ -61,6 +65,22 @@ def _load_numpy(path: Union[str, BinaryIO]) -> np.ndarray:
61
65
  return arr.astype(np.float32)
62
66
 
63
67
 
68
+ def _write_numpy(path: Union[str, BinaryIO], value: Any) -> None:
69
+ """Write an array to ``.npy`` or ``.npz`` based on extension."""
70
+ ensure_parent(path)
71
+ ext = os.path.splitext(_get_name(path))[1].lower()
72
+ arr = to_numpy(value).astype(np.float32)
73
+
74
+ if ext == _NPY_EXTENSION:
75
+ np.save(path, arr)
76
+ return
77
+ if ext == _NPZ_EXTENSION:
78
+ np.savez_compressed(path, data=arr)
79
+ return
80
+
81
+ raise ValueError(f"Unsupported output extension: {ext}")
82
+
83
+
64
84
  # ---------------------------------------------------------------------------
65
85
  # Public loaders
66
86
  # ---------------------------------------------------------------------------
@@ -86,6 +106,47 @@ def spherical_map(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None
86
106
  # ---------------------------------------------------------------------------
87
107
 
88
108
 
109
+ @modality_meta(
110
+ modality_type="intrinsics",
111
+ dtype="float32",
112
+ shape="3x3",
113
+ file_formats=[".npy", ".npz"],
114
+ )
115
+ def intrinsics(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> torch.Tensor:
116
+ """Load a camera intrinsics matrix as a ``(3, 3)`` float32 tensor.
117
+
118
+ The file is expected to contain a ``(3, 3)`` array::
119
+
120
+ [[fx, 0, cx],
121
+ [ 0, fy, cy],
122
+ [ 0, 0, 1]]
123
+ """
124
+ arr = _load_numpy(path)
125
+ return torch.from_numpy(arr).contiguous()
126
+
127
+
128
+ @modality_meta(
129
+ modality_type="sh_coeffs",
130
+ dtype="float32",
131
+ shape="NC",
132
+ file_formats=[".npy", ".npz"],
133
+ )
134
+ def sh_coeffs(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> torch.Tensor:
135
+ """Load spherical-harmonic coefficients as an ``(N, 3)`` float32 tensor.
136
+
137
+ *N* is the number of SH basis functions (e.g. 15 for degree-3 SH with
138
+ the constant term removed). Each row is a 3-vector (one per spatial
139
+ dimension).
140
+ """
141
+ arr = _load_numpy(path)
142
+ return torch.from_numpy(arr).contiguous()
143
+
144
+
145
+ # ---------------------------------------------------------------------------
146
+ # Writers
147
+ # ---------------------------------------------------------------------------
148
+
149
+
89
150
  @mark_stream_supported
90
151
  def write_spherical_map(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
91
152
  """Write spherical-map data to NumPy formats based on extension."""
@@ -101,3 +162,15 @@ def write_spherical_map(path: Union[str, BinaryIO], value: Any, meta: dict[str,
101
162
  return
102
163
 
103
164
  raise ValueError(f"Unsupported spherical-map output extension: {ext}")
165
+
166
+
167
+ @mark_stream_supported
168
+ def write_intrinsics(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
169
+ """Write a ``(3, 3)`` intrinsics matrix to a NumPy file."""
170
+ _write_numpy(path, value)
171
+
172
+
173
+ @mark_stream_supported
174
+ def write_sh_coeffs(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
175
+ """Write ``(N, 3)`` spherical-harmonic coefficients to a NumPy file."""
176
+ _write_numpy(path, value)