euler-loading 2.4.0__tar.gz → 2.5.0__tar.gz

{euler_loading-2.4.0 → euler_loading-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: euler-loading
-Version: 2.4.0
+Version: 2.5.0
 Summary: Multi-modal PyTorch dataloader using ds-crawler indices
 Requires-Python: >=3.9
 Requires-Dist: ds-crawler

{euler_loading-2.4.0 → euler_loading-2.5.0}/euler_loading/loaders/cpu/generic.py

@@ -10,6 +10,8 @@ Supported file extensions:
 
 Return types
 ------------
+- **map_2d** -- ``np.ndarray`` of shape ``(H, W)`` float32.
+- **map_3d** -- ``np.ndarray`` of shape ``(H, W, C)`` float32.
 - **spherical_map** -- ``np.ndarray`` of shape ``(H, W, C)`` float32.
 - **intrinsics** -- ``np.ndarray`` of shape ``(3, 3)`` float32.
 - **sh_coeffs** -- ``np.ndarray`` of shape ``(N, 3)`` float32.
@@ -19,6 +21,8 @@ Usage::
     from euler_loading.loaders.cpu import generic
     from euler_loading import Modality
 
+    Modality("/data/my_dataset/scattering_coefficient", loader=generic.map_2d)
+    Modality("/data/my_dataset/atmospheric_light", loader=generic.map_3d)
     Modality("/data/my_dataset/spherical_map", loader=generic.spherical_map)
     Modality("/data/my_dataset/intrinsics", loader=generic.intrinsics)
     Modality("/data/my_dataset/sh_coeffs", loader=generic.sh_coeffs)
@@ -36,6 +40,7 @@ from euler_loading.loaders._writer_utils import (
     ensure_parent,
     get_target_name,
     mark_stream_supported,
+    to_hw,
     to_numpy,
 )
 
@@ -85,6 +90,40 @@ def _write_numpy(path: Union[str, BinaryIO], value: Any) -> None:
 # ---------------------------------------------------------------------------
 
 
+@modality_meta(
+    modality_type="map_2d",
+    dtype="float32",
+    shape="HW",
+    file_formats=[".npy", ".npz"],
+)
+def map_2d(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> np.ndarray:
+    """Load an arbitrary 2D map as an ``(H, W)`` float32 array.
+
+    Suitable for any single-channel dense quantity (e.g. scattering
+    coefficient, attenuation, opacity) where no dataset-specific decoding
+    is required.
+    """
+    return _load_numpy(path)
+
+
+@modality_meta(
+    modality_type="map_3d",
+    dtype="float32",
+    shape="HWC",
+    file_formats=[".npy", ".npz"],
+)
+def map_3d(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> np.ndarray:
+    """Load an arbitrary 3D map as an ``(H, W, C)`` float32 array.
+
+    The file is expected to be stored in ``(C, H, W)`` layout and is
+    transposed to ``(H, W, C)`` for CPU-oriented processing.  Suitable
+    for any per-pixel vector quantity (e.g. atmospheric light, surface
+    normals, flow).
+    """
+    arr = _load_numpy(path)
+    return np.transpose(arr, (1, 2, 0))
+
+
 @modality_meta(
     modality_type="spherical_map",
     dtype="float32",
@@ -145,6 +184,39 @@ def sh_coeffs(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) ->
 # ---------------------------------------------------------------------------
 
 
+@mark_stream_supported
+def write_map_2d(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
+    """Write a 2D map to NumPy formats based on extension.
+
+    Accepts ``(H, W)``, ``(1, H, W)``, or ``(H, W, 1)`` input and stores
+    the array in ``(H, W)`` layout.
+    """
+    arr = to_hw(value, name="map_2d")
+    _write_numpy(path, arr)
+
+
+@mark_stream_supported
+def write_map_3d(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
+    """Write a 3D map to NumPy formats based on extension.
+
+    Input is expected in ``(H, W, C)`` layout and is transposed to
+    ``(C, H, W)`` for storage.
+    """
+    ensure_parent(path)
+    ext = os.path.splitext(_get_name(path))[1].lower()
+    arr = to_numpy(value).astype(np.float32)
+    arr = np.transpose(arr, (2, 0, 1))
+
+    if ext == _NPY_EXTENSION:
+        np.save(path, arr)
+        return
+    if ext == _NPZ_EXTENSION:
+        np.savez_compressed(path, data=arr)
+        return
+
+    raise ValueError(f"Unsupported map_3d output extension: {ext}")
+
+
 @mark_stream_supported
 def write_spherical_map(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
     """Write spherical-map data to NumPy formats based on extension.

{euler_loading-2.4.0 → euler_loading-2.5.0}/euler_loading/loaders/generate/loaders.json

@@ -3,6 +3,74 @@
         {
             "name": "generic",
             "modalities": [
+                {
+                    "type": "intrinsics",
+                    "function": "intrinsics",
+                    "hierarchical": false,
+                    "dtype": "float32",
+                    "access": "key",
+                    "file_formats": [
+                        ".npy",
+                        ".npz"
+                    ],
+                    "cpu": {
+                        "shape": "3x3"
+                    },
+                    "gpu": {
+                        "shape": "3x3"
+                    }
+                },
+                {
+                    "type": "map_2d",
+                    "function": "map_2d",
+                    "hierarchical": false,
+                    "dtype": "float32",
+                    "access": "key",
+                    "file_formats": [
+                        ".npy",
+                        ".npz"
+                    ],
+                    "cpu": {
+                        "shape": "HW"
+                    },
+                    "gpu": {
+                        "shape": "HW"
+                    }
+                },
+                {
+                    "type": "map_3d",
+                    "function": "map_3d",
+                    "hierarchical": false,
+                    "dtype": "float32",
+                    "access": "key",
+                    "file_formats": [
+                        ".npy",
+                        ".npz"
+                    ],
+                    "cpu": {
+                        "shape": "HWC"
+                    },
+                    "gpu": {
+                        "shape": "CHW"
+                    }
+                },
+                {
+                    "type": "sh_coeffs",
+                    "function": "sh_coeffs",
+                    "hierarchical": false,
+                    "dtype": "float32",
+                    "access": "key",
+                    "file_formats": [
+                        ".npy",
+                        ".npz"
+                    ],
+                    "cpu": {
+                        "shape": "NC"
+                    },
+                    "gpu": {
+                        "shape": "NC"
+                    }
+                },
                 {
                     "type": "spherical_map",
                     "function": "spherical_map",

{euler_loading-2.4.0 → euler_loading-2.5.0}/euler_loading/loaders/gpu/generic.py

@@ -10,6 +10,8 @@ Supported file extensions:
 
 Return types
 ------------
+- **map_2d** -- ``torch.FloatTensor`` of shape ``(H, W)``.
+- **map_3d** -- ``torch.FloatTensor`` of shape ``(C, H, W)``.
 - **spherical_map** -- ``torch.FloatTensor`` of shape ``(C, H, W)``.
 - **intrinsics** -- ``torch.FloatTensor`` of shape ``(3, 3)``.
 - **sh_coeffs** -- ``torch.FloatTensor`` of shape ``(N, 3)``.
@@ -19,6 +21,8 @@ Usage::
     from euler_loading.loaders.gpu import generic
     from euler_loading import Modality
 
+    Modality("/data/my_dataset/scattering_coefficient", loader=generic.map_2d)
+    Modality("/data/my_dataset/atmospheric_light", loader=generic.map_3d)
     Modality("/data/my_dataset/spherical_map", loader=generic.spherical_map)
     Modality("/data/my_dataset/intrinsics", loader=generic.intrinsics)
     Modality("/data/my_dataset/sh_coeffs", loader=generic.sh_coeffs)
@@ -37,6 +41,7 @@ from euler_loading.loaders._writer_utils import (
     ensure_parent,
     get_target_name,
     mark_stream_supported,
+    to_hw,
     to_numpy,
 )
 
@@ -86,6 +91,40 @@ def _write_numpy(path: Union[str, BinaryIO], value: Any) -> None:
 # ---------------------------------------------------------------------------
 
 
+@modality_meta(
+    modality_type="map_2d",
+    dtype="float32",
+    shape="HW",
+    file_formats=[".npy", ".npz"],
+)
+def map_2d(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> torch.Tensor:
+    """Load an arbitrary 2D map as an ``(H, W)`` float32 tensor.
+
+    Suitable for any single-channel dense quantity (e.g. scattering
+    coefficient, attenuation, opacity) where no dataset-specific decoding
+    is required.
+    """
+    arr = _load_numpy(path)
+    return torch.from_numpy(arr).contiguous()
+
+
+@modality_meta(
+    modality_type="map_3d",
+    dtype="float32",
+    shape="CHW",
+    file_formats=[".npy", ".npz"],
+)
+def map_3d(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) -> torch.Tensor:
+    """Load an arbitrary 3D map as a ``(C, H, W)`` float32 tensor.
+
+    The file is expected to already be stored in ``(C, H, W)`` layout,
+    matching the torch convention.  Suitable for any per-pixel vector
+    quantity (e.g. atmospheric light, surface normals, flow).
+    """
+    arr = _load_numpy(path)
+    return torch.from_numpy(arr).contiguous()
+
+
 @modality_meta(
     modality_type="spherical_map",
     dtype="float32",
@@ -147,6 +186,27 @@ def sh_coeffs(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None) ->
 # ---------------------------------------------------------------------------
 
 
+@mark_stream_supported
+def write_map_2d(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
+    """Write a 2D map to NumPy formats based on extension.
+
+    Accepts ``(H, W)``, ``(1, H, W)``, or ``(H, W, 1)`` input and stores
+    the array in ``(H, W)`` layout.
+    """
+    arr = to_hw(value, name="map_2d")
+    _write_numpy(path, arr)
+
+
+@mark_stream_supported
+def write_map_3d(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
+    """Write a 3D map to NumPy formats based on extension.
+
+    Input is expected in ``(C, H, W)`` layout (torch convention) and is
+    stored as-is.
+    """
+    _write_numpy(path, value)
+
+
 @mark_stream_supported
 def write_spherical_map(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
     """Write spherical-map data to NumPy formats based on extension."""

{euler_loading-2.4.0 → euler_loading-2.5.0}/package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "loading-better-save-outputs",
+  "name": "loading-2D-map",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {}

{euler_loading-2.4.0 → euler_loading-2.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "euler-loading"
-version = "2.4.0"
+version = "2.5.0"
 description = "Multi-modal PyTorch dataloader using ds-crawler indices"
 requires-python = ">=3.9"
 dependencies = [

{euler_loading-2.4.0 → euler_loading-2.5.0}/tests/test_loaders.py

@@ -485,8 +485,8 @@ class TestRDSWriters:
 # Generic spherical_map loader tests
 # ---------------------------------------------------------------------------
 
-GENERIC_LOADER_NAMES = ["spherical_map"]
-GENERIC_WRITER_NAMES = ["write_spherical_map"]
+GENERIC_LOADER_NAMES = ["map_2d", "map_3d", "spherical_map"]
+GENERIC_WRITER_NAMES = ["write_map_2d", "write_map_3d", "write_spherical_map"]
 
 
 @pytest.fixture()
@@ -638,3 +638,187 @@ class TestGenericBackwardCompat:
         top = generic_top.spherical_map(path)
         gpu = gpu_generic.spherical_map(path)
         assert torch.equal(top, gpu)
+
+
+# ---------------------------------------------------------------------------
+# Generic map_2d / map_3d loader tests
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture()
+def map_2d_npy_path(tmp_path):
+    """Write a small (4, 5) float32 .npy file."""
+    arr = np.random.default_rng(0).random((4, 5), dtype=np.float32)
+    p = tmp_path / "map_2d.npy"
+    np.save(str(p), arr)
+    return str(p), arr
+
+
+@pytest.fixture()
+def map_2d_npz_path(tmp_path):
+    """Write a small (4, 5) float32 .npz file."""
+    arr = np.random.default_rng(1).random((4, 5), dtype=np.float32)
+    p = tmp_path / "map_2d.npz"
+    np.savez_compressed(str(p), data=arr)
+    return str(p), arr
+
+
+@pytest.fixture()
+def map_3d_npy_path(tmp_path):
+    """Write a small (2, 4, 5) float32 .npy file in CHW layout."""
+    arr = np.random.default_rng(2).random((2, 4, 5), dtype=np.float32)
+    p = tmp_path / "map_3d.npy"
+    np.save(str(p), arr)
+    return str(p), arr
+
+
+@pytest.fixture()
+def map_3d_npz_path(tmp_path):
+    """Write a small (2, 4, 5) float32 .npz file in CHW layout."""
+    arr = np.random.default_rng(3).random((2, 4, 5), dtype=np.float32)
+    p = tmp_path / "map_3d.npz"
+    np.savez_compressed(str(p), data=arr)
+    return str(p), arr
+
+
+class TestGPUMap2DLoader:
+    """GPU ``map_2d`` loads a 2D map as an ``(H, W)`` float32 tensor."""
+
+    def test_npy_dtype_and_shape(self, map_2d_npy_path):
+        path, _ = map_2d_npy_path
+        result = gpu_generic.map_2d(path)
+        assert isinstance(result, torch.Tensor)
+        assert result.dtype == torch.float32
+        assert result.shape == (4, 5)
+
+    def test_npy_values_match(self, map_2d_npy_path):
+        path, expected = map_2d_npy_path
+        result = gpu_generic.map_2d(path)
+        assert torch.allclose(result, torch.from_numpy(expected))
+
+    def test_npz_dtype_and_shape(self, map_2d_npz_path):
+        path, _ = map_2d_npz_path
+        result = gpu_generic.map_2d(path)
+        assert result.dtype == torch.float32
+        assert result.shape == (4, 5)
+
+
+class TestCPUMap2DLoader:
+    """CPU ``map_2d`` loads a 2D map as an ``(H, W)`` float32 array."""
+
+    def test_npy_dtype_and_shape(self, map_2d_npy_path):
+        path, _ = map_2d_npy_path
+        result = cpu_generic.map_2d(path)
+        assert isinstance(result, np.ndarray)
+        assert result.dtype == np.float32
+        assert result.shape == (4, 5)
+
+    def test_npy_values_match(self, map_2d_npy_path):
+        path, expected = map_2d_npy_path
+        result = cpu_generic.map_2d(path)
+        assert np.allclose(result, expected)
+
+    def test_npz_dtype_and_shape(self, map_2d_npz_path):
+        path, _ = map_2d_npz_path
+        result = cpu_generic.map_2d(path)
+        assert result.dtype == np.float32
+        assert result.shape == (4, 5)
+
+
+class TestGPUMap3DLoader:
+    """GPU ``map_3d`` loads a 3D map as a ``(C, H, W)`` float32 tensor."""
+
+    def test_npy_dtype_and_shape(self, map_3d_npy_path):
+        path, _ = map_3d_npy_path
+        result = gpu_generic.map_3d(path)
+        assert isinstance(result, torch.Tensor)
+        assert result.dtype == torch.float32
+        assert result.shape == (2, 4, 5)
+
+    def test_npy_values_match(self, map_3d_npy_path):
+        path, expected = map_3d_npy_path
+        result = gpu_generic.map_3d(path)
+        assert torch.allclose(result, torch.from_numpy(expected))
+
+    def test_npz_dtype_and_shape(self, map_3d_npz_path):
+        path, _ = map_3d_npz_path
+        result = gpu_generic.map_3d(path)
+        assert result.shape == (2, 4, 5)
+
+
+class TestCPUMap3DLoader:
+    """CPU ``map_3d`` transposes CHW-on-disk to ``(H, W, C)``."""
+
+    def test_npy_dtype_and_shape(self, map_3d_npy_path):
+        path, _ = map_3d_npy_path
+        result = cpu_generic.map_3d(path)
+        assert isinstance(result, np.ndarray)
+        assert result.dtype == np.float32
+        assert result.shape == (4, 5, 2)
+
+    def test_npy_values_match(self, map_3d_npy_path):
+        path, expected = map_3d_npy_path
+        result = cpu_generic.map_3d(path)
+        assert np.allclose(result, np.transpose(expected, (1, 2, 0)))
+
+
+class TestMapWriters:
+    """Writer round-trip tests for ``map_2d`` / ``map_3d``."""
+
+    def test_gpu_write_map_2d_npy_roundtrip(self, tmp_path):
+        data = torch.rand(4, 5, dtype=torch.float32)
+        path = tmp_path / "m.npy"
+        gpu_generic.write_map_2d(str(path), data)
+        loaded = gpu_generic.map_2d(str(path))
+        assert torch.allclose(loaded, data)
+
+    def test_gpu_write_map_2d_accepts_1hw(self, tmp_path):
+        data = torch.rand(1, 4, 5, dtype=torch.float32)
+        path = tmp_path / "m.npy"
+        gpu_generic.write_map_2d(str(path), data)
+        loaded = gpu_generic.map_2d(str(path))
+        assert loaded.shape == (4, 5)
+        assert torch.allclose(loaded, data.squeeze(0))
+
+    def test_gpu_write_map_2d_npz_roundtrip(self, tmp_path):
+        data = torch.rand(4, 5, dtype=torch.float32)
+        path = tmp_path / "m.npz"
+        gpu_generic.write_map_2d(str(path), data)
+        loaded = gpu_generic.map_2d(str(path))
+        assert torch.allclose(loaded, data)
+
+    def test_cpu_write_map_2d_npy_roundtrip(self, tmp_path):
+        data = np.random.default_rng(0).random((4, 5)).astype(np.float32)
+        path = tmp_path / "m.npy"
+        cpu_generic.write_map_2d(str(path), data)
+        loaded = cpu_generic.map_2d(str(path))
+        assert np.allclose(loaded, data)
+
+    def test_gpu_write_map_3d_npy_roundtrip(self, tmp_path):
+        data = torch.rand(2, 4, 5, dtype=torch.float32)
+        path = tmp_path / "m.npy"
+        gpu_generic.write_map_3d(str(path), data)
+        loaded = gpu_generic.map_3d(str(path))
+        assert torch.allclose(loaded, data)
+
+    def test_cpu_write_map_3d_npy_roundtrip(self, tmp_path):
+        data = np.random.default_rng(0).random((4, 5, 2)).astype(np.float32)
+        path = tmp_path / "m.npy"
+        cpu_generic.write_map_3d(str(path), data)
+        loaded = cpu_generic.map_3d(str(path))
+        assert np.allclose(loaded, data)
+
+    def test_cpu_write_map_3d_npz_roundtrip(self, tmp_path):
+        data = np.random.default_rng(0).random((4, 5, 2)).astype(np.float32)
+        path = tmp_path / "m.npz"
+        cpu_generic.write_map_3d(str(path), data)
+        loaded = cpu_generic.map_3d(str(path))
+        assert np.allclose(loaded, data)
+
+    def test_gpu_to_cpu_map_3d_matches_transpose(self, tmp_path):
+        """A GPU-written map_3d reads back on CPU as the HWC transpose."""
+        data = torch.rand(2, 4, 5, dtype=torch.float32)
+        path = tmp_path / "m.npy"
+        gpu_generic.write_map_3d(str(path), data)
+        loaded = cpu_generic.map_3d(str(path))
+        assert np.allclose(loaded, data.permute(1, 2, 0).numpy())