euler-loading 1.3.0__tar.gz → 2.1.0__tar.gz

{euler_loading-1.3.0 → euler_loading-2.1.0}/PKG-INFO

@@ -1,9 +1,10 @@
 Metadata-Version: 2.4
 Name: euler-loading
-Version: 1.3.0
+Version: 2.1.0
 Summary: Multi-modal PyTorch dataloader using ds-crawler indices
 Requires-Python: >=3.9
 Requires-Dist: ds-crawler
+Requires-Dist: euler-dataset-contract
 Requires-Dist: numpy
 Requires-Dist: pillow
 Provides-Extra: dev

{euler_loading-1.3.0 → euler_loading-2.1.0}/euler_loading/__init__.py

@@ -1,5 +1,6 @@
 """euler-loading: Multi-modal PyTorch dataloader using ds-crawler indices."""
 
+from . import _dataset_contract  # noqa: F401
 from ._resolution import resolve_loader_module, resolve_writer_module
 from ._writing import create_dataset_writer_from_index
 from .dataset import Modality, MultiModalDataset

euler_loading-2.1.0/euler_loading/_dataset_contract.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from typing import Any
+
+from euler_dataset_contract import (
+    register_addon_validator,
+    validate_addon_version,
+    validate_slot,
+    validate_string_list,
+    validate_token,
+)
+
+
+_ALLOWED_USED_AS = {"condition", "input", "output", "target"}
+_ALLOWED_KEYS = {
+    "version",
+    "used_as",
+    "slot",
+    "hierarchy_scope",
+    "applies_to",
+    "task",
+    "loader",
+    "function",
+    "writer_function",
+}
+
+
+def validate_euler_loading_addon(value: Any, context: str) -> None:
+    if not isinstance(value, dict):
+        raise ValueError(f"{context} must be an object")
+
+    unknown = sorted(set(value.keys()) - _ALLOWED_KEYS)
+    if unknown:
+        joined = ", ".join(unknown)
+        raise ValueError(f"Unknown {context} key(s): {joined}")
+
+    validate_addon_version(value.get("version"), f"{context}.version")
+
+    used_as = value.get("used_as")
+    if used_as is not None:
+        if not isinstance(used_as, str) or not used_as:
+            raise ValueError(f"{context}.used_as must be a non-empty string")
+        if used_as not in _ALLOWED_USED_AS:
+            allowed = ", ".join(sorted(_ALLOWED_USED_AS))
+            raise ValueError(
+                f"{context}.used_as must be one of {{{allowed}}}, got {used_as!r}"
+            )
+
+    slot = value.get("slot")
+    if slot is not None:
+        validate_slot(slot, f"{context}.slot")
+
+    task = value.get("task")
+    if task is not None:
+        validate_token(task, f"{context}.task")
+
+    loader = value.get("loader")
+    function = value.get("function")
+    if loader is not None:
+        validate_token(loader, f"{context}.loader")
+    if function is not None:
+        validate_token(function, f"{context}.function")
+    if (loader is None) != (function is None):
+        raise ValueError(
+            f"{context}.loader and {context}.function must either both be set "
+            "or both be omitted"
+        )
+
+    writer_function = value.get("writer_function")
+    if writer_function is not None:
+        validate_token(writer_function, f"{context}.writer_function")
+
+    hierarchy_scope = value.get("hierarchy_scope")
+    applies_to = value.get("applies_to")
+    if hierarchy_scope is not None or applies_to is not None:
+        if used_as != "condition":
+            raise ValueError(
+                f"{context}.hierarchy_scope and {context}.applies_to are only "
+                "allowed when used_as is 'condition'"
+            )
+        if hierarchy_scope is not None:
+            validate_token(hierarchy_scope, f"{context}.hierarchy_scope")
+        if applies_to is not None:
+            validate_string_list(
+                applies_to,
+                f"{context}.applies_to",
+                allow_wildcard=True,
+            )
+
+
+register_addon_validator("euler_loading", validate_euler_loading_addon, overwrite=True)

{euler_loading-1.3.0 → euler_loading-2.1.0}/euler_loading/_ds_crawler_utils.py

@@ -5,6 +5,7 @@ from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
 
+from ds_crawler import get_dataset_contract
 from ds_crawler.zip_utils import read_metadata_json
 
 try:
@@ -13,19 +14,6 @@ except ImportError:
     _load_dataset_split = None
 
 
-DS_CRAWLER_STRUCTURAL_KEYS = frozenset({
-    "name",
-    "type",
-    "id_regex",
-    "id_regex_join_char",
-    "euler_train",
-    "dataset",
-    "hierarchy_regex",
-    "named_capture_group_value_separator",
-    "sampled",
-})
-
-
 def as_non_empty_str(value: Any) -> str | None:
     if value is None:
         return None
@@ -61,11 +49,7 @@ def first_non_empty_list(*candidates: list[str] | None) -> list[str]:
 
 
 def extract_ds_crawler_properties(index_output: Mapping[str, Any]) -> dict[str, Any]:
-    return {
-        str(key): value
-        for key, value in index_output.items()
-        if key not in DS_CRAWLER_STRUCTURAL_KEYS
-    }
+    return get_dataset_contract(dict(index_output)).to_properties_dict()
 
 
 _SPLIT_NAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
@@ -164,5 +148,6 @@ def load_index_output(
         )
 
     result = dict(base_output)
-    result["dataset"] = split_dataset
+    result["index"] = split_dataset
+    result.pop("dataset", None)
     return result

{euler_loading-1.3.0 → euler_loading-2.1.0}/euler_loading/_metadata.py

@@ -4,6 +4,8 @@ import re
 from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Callable
 
+from ds_crawler import get_dataset_contract
+
 from ._ds_crawler_utils import (
     as_non_empty_str,
     as_string_list,
@@ -44,6 +46,7 @@ def _build_runlog_entry(
         as_non_empty_str(
             _resolve_euler_loading_property(euler_loading_properties, "modality_type")
         ),
+        as_non_empty_str(descriptor.get("modality_key")),
         as_non_empty_str(descriptor.get("modality_type")),
         _infer_modality_type(name=name, path=modality.path),
     )
@@ -146,12 +149,19 @@ def _read_ds_crawler_descriptor(
 
     if isinstance(index_output, Mapping):
         properties.update(extract_ds_crawler_properties(index_output))
-        index_type = as_non_empty_str(index_output.get("type"))
-        if index_type is not None:
-            descriptor["modality_type"] = index_type
-        hierarchy_regex = as_non_empty_str(index_output.get("hierarchy_regex"))
-        if hierarchy_regex is not None:
-            descriptor["hierarchy_regex"] = hierarchy_regex
+        try:
+            contract = get_dataset_contract(dict(index_output))
+        except Exception:
+            contract = None
+        if contract is not None:
+            descriptor["modality_key"] = contract.modality_key
+        indexing = index_output.get("indexing")
+        if isinstance(indexing, Mapping):
+            hierarchy = indexing.get("hierarchy")
+            if isinstance(hierarchy, Mapping):
+                hierarchy_regex = as_non_empty_str(hierarchy.get("regex"))
+                if hierarchy_regex is not None:
+                    descriptor["hierarchy_regex"] = hierarchy_regex
 
     try:
         cfg = load_dataset_config_fn({"path": path})
@@ -164,7 +174,7 @@ def _read_ds_crawler_descriptor(
 
     cfg_type = as_non_empty_str(getattr(cfg, "type", None))
     if cfg_type is not None:
-        descriptor["modality_type"] = cfg_type
+        descriptor["modality_key"] = cfg_type
 
     cfg_hierarchy_regex = as_non_empty_str(getattr(cfg, "hierarchy_regex", None))
     if cfg_hierarchy_regex is not None:

{euler_loading-1.3.0 → euler_loading-2.1.0}/euler_loading/_resolution.py

@@ -6,6 +6,8 @@ from collections.abc import Mapping
 from types import ModuleType
 from typing import TYPE_CHECKING, Any, Callable
 
+from ds_crawler import get_dataset_contract
+
 from ._ds_crawler_utils import as_non_empty_str
 
 if TYPE_CHECKING:
@@ -61,7 +63,11 @@ def _resolve_loader(
     if modality.loader is not None:
         return modality.loader
 
-    euler_loading_meta = index.get("euler_loading")
+    try:
+        contract = get_dataset_contract(index)
+        euler_loading_meta = contract.get_addon("euler_loading")
+    except Exception:
+        euler_loading_meta = None
     if not isinstance(euler_loading_meta, Mapping):
         raise ValueError(
             f"Modality {modality_name!r}: no explicit loader provided and the "
@@ -126,7 +132,11 @@ def _resolve_writer(
     if modality.writer is not None:
         return modality.writer
 
-    euler_loading_meta = index.get("euler_loading")
+    try:
+        contract = get_dataset_contract(index)
+        euler_loading_meta = contract.get_addon("euler_loading")
+    except Exception:
+        euler_loading_meta = None
     if not isinstance(euler_loading_meta, Mapping):
         return None
 

{euler_loading-1.3.0 → euler_loading-2.1.0}/euler_loading/_writing.py

@@ -7,11 +7,10 @@ from collections.abc import Mapping
 from pathlib import Path
 from typing import Any, Callable
 
-from ds_crawler import DatasetWriter, ZipDatasetWriter
+from ds_crawler import DatasetWriter, ZipDatasetWriter, get_dataset_contract
 
 from ._ds_crawler_utils import (
     as_non_empty_str,
-    extract_ds_crawler_properties,
     first_non_empty,
 )
 from .loaders._writer_utils import supports_stream_target
@@ -30,27 +29,20 @@ def create_dataset_writer_from_index(
     zip: bool = False,
 ) -> DatasetWriter | ZipDatasetWriter:
     """Create a ds-crawler writer that mirrors an existing index's metadata."""
-    name = as_non_empty_str(index_output.get("name")) or "dataset"
-    type_name = as_non_empty_str(index_output.get("type")) or "other"
-    euler_train = index_output.get("euler_train")
-    if not isinstance(euler_train, Mapping):
-        raise ValueError(
-            "index_output must contain an 'euler_train' mapping to build a writer."
-        )
+    contract = get_dataset_contract(dict(index_output))
 
+    indexing = index_output.get("indexing")
+    hierarchy = indexing.get("hierarchy") if isinstance(indexing, Mapping) else None
+    id_cfg = indexing.get("id") if isinstance(indexing, Mapping) else None
     separator = first_non_empty(
-        as_non_empty_str(index_output.get("named_capture_group_value_separator")),
-        as_non_empty_str(index_output.get("id_regex_join_char")),
+        as_non_empty_str(hierarchy.get("separator")) if isinstance(hierarchy, Mapping) else None,
+        as_non_empty_str(id_cfg.get("join_char")) if isinstance(id_cfg, Mapping) else None,
     )
-    properties = extract_ds_crawler_properties(index_output)
     writer_cls = ZipDatasetWriter if zip else DatasetWriter
     return writer_cls(
         root,
-        name=name,
-        type=type_name,
-        euler_train=dict(euler_train),
+        head=contract.to_mapping(),
         separator=separator,
-        **properties,
     )
 
 

{euler_loading-1.3.0 → euler_loading-2.1.0}/euler_loading/dataset.py

@@ -18,6 +18,7 @@ except ImportError:
 from ds_crawler import (
     DatasetWriter,
     ZipDatasetWriter,
+    get_dataset_contract,
     index_dataset_from_path,
     load_dataset_config,
 )
@@ -319,7 +320,7 @@ class MultiModalDataset(_BaseDataset):
             self._resolved_writers[name] = _resolve_writer(
                 modality_name=name, modality=modality, index=index,
             )
-            records = collect_files(index["dataset"])
+            records = collect_files(index["index"])
             lookup = {
                 "/".join(rec.hierarchy_path + (rec.file_entry["id"],)): rec
                 for rec in records
@@ -351,7 +352,7 @@ class MultiModalDataset(_BaseDataset):
             self._resolved_writers[name] = _resolve_writer(
                 modality_name=name, modality=modality, index=index,
             )
-            files_by_level = collect_hierarchical_files(index["dataset"])
+            files_by_level = collect_hierarchical_files(index["index"])
             self._hierarchical_lookups[name] = files_by_level
             total_files = sum(len(f) for f in files_by_level.values())
             logger.info(
@@ -407,7 +408,11 @@ class MultiModalDataset(_BaseDataset):
 
     def get_modality_metadata(self, modality_name: str) -> dict[str, Any]:
         """Return the metadata dict for a given modality name."""
-        return self._index_outputs.get(modality_name, {}).get("meta", {})
+        index = self._index_outputs.get(modality_name, {})
+        try:
+            return get_dataset_contract(index).meta or {}
+        except Exception:
+            return {}
 
     def get_modality_index(self, modality_name: str) -> dict[str, Any]:
         """Return the cached ds-crawler index for a modality."""
@@ -505,7 +510,12 @@ class MultiModalDataset(_BaseDataset):
                 output_root=output_root, modality_name=modality_name
             )
             record = self._lookups[modality_name][sample_id]
-            modality_meta = self._index_outputs[modality_name].get("meta")
+            try:
+                modality_meta = get_dataset_contract(
+                    self._index_outputs[modality_name]
+                ).meta
+            except Exception:
+                modality_meta = None
             basename = Path(record.file_entry["path"]).name
             relative_path = record.file_entry["path"]
             full_id = _build_writer_full_id(
@@ -545,7 +555,10 @@ class MultiModalDataset(_BaseDataset):
         first_modality: str | None = None
 
         for modality_name, index in self._index_outputs.items():
-            name = index.get("name")
+            try:
+                name = get_dataset_contract(index).name
+            except Exception:
+                name = None
             if name is None:
                 continue
             if first_name is None:
@@ -604,7 +617,12 @@ class MultiModalDataset(_BaseDataset):
 
         for name, modality in self._modalities.items():
             record = self._lookups[name][sample_id]
-            modality_meta = self._index_outputs[name].get("meta")
+            try:
+                modality_meta = get_dataset_contract(
+                    self._index_outputs[name]
+                ).meta
+            except Exception:
+                modality_meta = None
 
             if name in self._zip_modalities:
                 file_or_path = self._open_from_zip(
@@ -625,7 +643,12 @@ class MultiModalDataset(_BaseDataset):
         for name, modality in self._hierarchical_modalities.items():
             files_by_level = self._hierarchical_lookups[name]
             matched = match_hierarchical_files(hierarchy_path, files_by_level)
-            modality_meta = self._hierarchical_index_outputs[name].get("meta")
+            try:
+                modality_meta = get_dataset_contract(
+                    self._hierarchical_index_outputs[name]
+                ).meta
+            except Exception:
+                modality_meta = None
             loaded: dict[str, Any] = {}
             for entry in matched:
                 cache_key = f"{modality.path}/{entry['path']}"

{euler_loading-1.3.0 → euler_loading-2.1.0}/euler_loading/loaders/cpu/generic.py

@@ -11,6 +11,8 @@ Supported file extensions:
 Return types
 ------------
 - **spherical_map** -- ``np.ndarray`` of shape ``(H, W, C)`` float32.
+- **intrinsics** -- ``np.ndarray`` of shape ``(3, 3)`` float32.
+- **sh_coeffs** -- ``np.ndarray`` of shape ``(N, 3)`` float32.
 
 Usage::
 
@@ -18,6 +20,8 @@ Usage::
     from euler_loading import Modality
 
     Modality("/data/my_dataset/spherical_map", loader=generic.spherical_map)
+    Modality("/data/my_dataset/intrinsics", loader=generic.intrinsics)
+    Modality("/data/my_dataset/sh_coeffs", loader=generic.sh_coeffs)
 """
 
 from __future__ import annotations
@@ -60,6 +64,22 @@ def _load_numpy(path: Union[str, BinaryIO]) -> np.ndarray:
     return arr.astype(np.float32)
 
 
+def _write_numpy(path: Union[str, BinaryIO], value: Any) -> None:
+    """Write an array to ``.npy`` or ``.npz`` based on extension."""
+    ensure_parent(path)
+    ext = os.path.splitext(_get_name(path))[1].lower()
+    arr = to_numpy(value).astype(np.float32)
+
+    if ext == _NPY_EXTENSION:
+        np.save(path, arr)
+        return
+    if ext == _NPZ_EXTENSION:
+        np.savez_compressed(path, data=arr)
+        return
+
+    raise ValueError(f"Unsupported output extension: {ext}")
+
+
 # ---------------------------------------------------------------------------
 # Public loaders
 # ---------------------------------------------------------------------------
@@ -86,6 +106,45 @@ def spherical_map(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None
 # ---------------------------------------------------------------------------
 
 
+@modality_meta(
+    modality_type="intrinsics",
+    dtype="float32",
+    shape="3x3",
+    file_formats=[".npy", ".npz"],
+)
+def intrinsics(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> np.ndarray:
+    """Load a camera intrinsics matrix as a ``(3, 3)`` float32 array.
+
+    The file is expected to contain a ``(3, 3)`` array::
+
+        [[fx,  0, cx],
+         [ 0, fy, cy],
+         [ 0,  0,  1]]
+    """
+    return _load_numpy(path)
+
+
+@modality_meta(
+    modality_type="sh_coeffs",
+    dtype="float32",
+    shape="NC",
+    file_formats=[".npy", ".npz"],
+)
+def sh_coeffs(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> np.ndarray:
+    """Load spherical-harmonic coefficients as an ``(N, 3)`` float32 array.
+
+    *N* is the number of SH basis functions (e.g. 15 for degree-3 SH with
+    the constant term removed).  Each row is a 3-vector (one per spatial
+    dimension).
+    """
+    return _load_numpy(path)
+
+
+# ---------------------------------------------------------------------------
+# Writers
+# ---------------------------------------------------------------------------
+
+
 @mark_stream_supported
 def write_spherical_map(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
     """Write spherical-map data to NumPy formats based on extension.
@@ -107,3 +166,15 @@ def write_spherical_map(path: Union[str, BinaryIO], value: Any, meta: dict[str,
         return
 
     raise ValueError(f"Unsupported spherical-map output extension: {ext}")
+
+
+@mark_stream_supported
+def write_intrinsics(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
+    """Write a ``(3, 3)`` intrinsics matrix to a NumPy file."""
+    _write_numpy(path, value)
+
+
+@mark_stream_supported
+def write_sh_coeffs(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
+    """Write ``(N, 3)`` spherical-harmonic coefficients to a NumPy file."""
+    _write_numpy(path, value)

{euler_loading-1.3.0 → euler_loading-2.1.0}/euler_loading/loaders/gpu/generic.py

@@ -11,6 +11,8 @@ Supported file extensions:
 Return types
 ------------
 - **spherical_map** -- ``torch.FloatTensor`` of shape ``(C, H, W)``.
+- **intrinsics** -- ``torch.FloatTensor`` of shape ``(3, 3)``.
+- **sh_coeffs** -- ``torch.FloatTensor`` of shape ``(N, 3)``.
 
 Usage::
 
@@ -18,6 +20,8 @@ Usage::
     from euler_loading import Modality
 
     Modality("/data/my_dataset/spherical_map", loader=generic.spherical_map)
+    Modality("/data/my_dataset/intrinsics", loader=generic.intrinsics)
+    Modality("/data/my_dataset/sh_coeffs", loader=generic.sh_coeffs)
 """
 
 from __future__ import annotations
@@ -61,6 +65,22 @@ def _load_numpy(path: Union[str, BinaryIO]) -> np.ndarray:
     return arr.astype(np.float32)
 
 
+def _write_numpy(path: Union[str, BinaryIO], value: Any) -> None:
+    """Write an array to ``.npy`` or ``.npz`` based on extension."""
+    ensure_parent(path)
+    ext = os.path.splitext(_get_name(path))[1].lower()
+    arr = to_numpy(value).astype(np.float32)
+
+    if ext == _NPY_EXTENSION:
+        np.save(path, arr)
+        return
+    if ext == _NPZ_EXTENSION:
+        np.savez_compressed(path, data=arr)
+        return
+
+    raise ValueError(f"Unsupported output extension: {ext}")
+
+
 # ---------------------------------------------------------------------------
 # Public loaders
 # ---------------------------------------------------------------------------
@@ -86,6 +106,47 @@ def spherical_map(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None
 # ---------------------------------------------------------------------------
 
 
+@modality_meta(
+    modality_type="intrinsics",
+    dtype="float32",
+    shape="3x3",
+    file_formats=[".npy", ".npz"],
+)
+def intrinsics(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> torch.Tensor:
+    """Load a camera intrinsics matrix as a ``(3, 3)`` float32 tensor.
+
+    The file is expected to contain a ``(3, 3)`` array::
+
+        [[fx,  0, cx],
+         [ 0, fy, cy],
+         [ 0,  0,  1]]
+    """
+    arr = _load_numpy(path)
+    return torch.from_numpy(arr).contiguous()
+
+
+@modality_meta(
+    modality_type="sh_coeffs",
+    dtype="float32",
+    shape="NC",
+    file_formats=[".npy", ".npz"],
+)
+def sh_coeffs(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> torch.Tensor:
+    """Load spherical-harmonic coefficients as an ``(N, 3)`` float32 tensor.
+
+    *N* is the number of SH basis functions (e.g. 15 for degree-3 SH with
+    the constant term removed).  Each row is a 3-vector (one per spatial
+    dimension).
+    """
+    arr = _load_numpy(path)
+    return torch.from_numpy(arr).contiguous()
+
+
+# ---------------------------------------------------------------------------
+# Writers
+# ---------------------------------------------------------------------------
+
+
 @mark_stream_supported
 def write_spherical_map(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
     """Write spherical-map data to NumPy formats based on extension."""
@@ -101,3 +162,15 @@ def write_spherical_map(path: Union[str, BinaryIO], value: Any, meta: dict[str,
         return
 
     raise ValueError(f"Unsupported spherical-map output extension: {ext}")
+
+
+@mark_stream_supported
+def write_intrinsics(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
+    """Write a ``(3, 3)`` intrinsics matrix to a NumPy file."""
+    _write_numpy(path, value)
+
+
+@mark_stream_supported
+def write_sh_coeffs(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
+    """Write ``(N, 3)`` spherical-harmonic coefficients to a NumPy file."""
+    _write_numpy(path, value)

{euler_loading-1.3.0 → euler_loading-2.1.0}/pyproject.toml

@@ -1,12 +1,13 @@
 [project]
 name = "euler-loading"
-version = "1.3.0"
+version = "2.1.0"
 description = "Multi-modal PyTorch dataloader using ds-crawler indices"
 requires-python = ">=3.9"
 dependencies = [
     "numpy",
     "Pillow",
     "ds-crawler",
+    "euler-dataset-contract",
 ]
 
 [project.optional-dependencies]