euler-loading 2.18.0__tar.gz → 2.19.0__tar.gz

{euler_loading-2.18.0 → euler_loading-2.19.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: euler-loading
-Version: 2.18.0
+Version: 2.19.0
 Summary: Multi-modal PyTorch dataloader using ds-crawler indices
 Requires-Python: >=3.9
 Requires-Dist: ds-crawler>=2.3.0

{euler_loading-2.18.0 → euler_loading-2.19.0}/README.md

@@ -190,6 +190,24 @@ when doing so is unambiguous:
 
 This is an additive layout. Existing roots with `.ds_crawler/index.json` continue to load unchanged, and a scoped modality falls back to that legacy location when the scoped artifacts are not present.
 
+Output writers can create the same layout natively. Pass a scope when creating
+each modality writer; ds-crawler writes the scoped artifacts and maintains
+`.ds_crawler/scopes.json`:
+
+```python
+rgb_writer = dataset.create_output_writer(
+    "rgb", output_root, metadata_scope="rgb"
+)
+depth_writer = dataset.create_output_writer(
+    "sparse_depth", output_root, metadata_scope="sparse_depth"
+)
+```
+
+The standalone `create_dataset_writer_from_index(...)` helper accepts the same
+`metadata_scope` argument. For ZIP output, one `ZipDatasetWriter` owns and
+finalizes one archive, so do not open multiple ZIP writers concurrently for the
+same path.
+
 ### `MultiModalDataset.modality_paths()`
 
 Returns a dict mapping each regular modality name to `{"path": ..., "origin_path": ...}` and includes `split` and `metadata_scope` when configured.

{euler_loading-2.18.0 → euler_loading-2.19.0}/euler_loading/_writing.py

@@ -28,6 +28,8 @@ OutputDestination = str | os.PathLike[str] | DatasetWriter | ZipDatasetWriter
 
 def _wrap_writer_save_index(
     writer: DatasetWriter | ZipDatasetWriter,
+    *,
+    metadata_scope: str | None = None,
 ) -> DatasetWriter | ZipDatasetWriter:
     original_save_index = writer.save_index
 
@@ -72,7 +74,12 @@ def _wrap_writer_save_index(
         path = original_save_index(filename=filename)
 
         if isinstance(writer, ZipDatasetWriter):
-            entry_name = f".ds_crawler/{filename}"
+            metadata_prefix = (
+                f".ds_crawler/{metadata_scope}"
+                if metadata_scope is not None
+                else ".ds_crawler"
+            )
+            entry_name = f"{metadata_prefix}/{filename}"
             archive_path = Path(writer.root)
             with zipfile.ZipFile(archive_path, "r") as source_zip:
                 output_index = json.loads(source_zip.read(entry_name).decode("utf-8"))
@@ -113,8 +120,15 @@ def create_dataset_writer_from_index(
     index_output: Mapping[str, Any],
     root: str | os.PathLike[str],
     zip: bool = False,
+    metadata_scope: str | None = None,
 ) -> DatasetWriter | ZipDatasetWriter:
-    """Create a ds-crawler writer that mirrors an existing index's metadata."""
+    """Create a ds-crawler writer that mirrors an existing index's metadata.
+
+    ``metadata_scope`` stores the generated artifacts below
+    ``.ds_crawler/<metadata_scope>/`` and lets ds-crawler maintain the shared
+    ``.ds_crawler/scopes.json`` manifest.  This is useful when several logical
+    modalities share one physical output root or archive.
+    """
     indexing = index_output.get("indexing")
     hierarchy = indexing.get("hierarchy") if isinstance(indexing, Mapping) else None
     id_cfg = indexing.get("id") if isinstance(indexing, Mapping) else None
@@ -133,8 +147,9 @@ def create_dataset_writer_from_index(
             root,
             head=contract.to_mapping(),
             separator=separator,
+            metadata_scope=metadata_scope,
         )
-        return _wrap_writer_save_index(writer)
+        return _wrap_writer_save_index(writer, metadata_scope=metadata_scope)
 
     name = as_non_empty_str(index_output.get("name"))
     type_name = as_non_empty_str(index_output.get("type"))
@@ -160,9 +175,10 @@ def create_dataset_writer_from_index(
         type=type_name,
         euler_train=dict(euler_train) if isinstance(euler_train, Mapping) else None,
         separator=separator,
+        metadata_scope=metadata_scope,
         **properties,
     )
-    return _wrap_writer_save_index(writer)
+    return _wrap_writer_save_index(writer, metadata_scope=metadata_scope)
 
 
 def _resolve_output_destination(

{euler_loading-2.18.0 → euler_loading-2.19.0}/euler_loading/dataset.py

@@ -969,13 +969,23 @@ class MultiModalDataset(_BaseDataset):
         root: str | os.PathLike[str],
         *,
         zip: bool = False,
+        metadata_scope: str | None = None,
     ) -> DatasetWriter | ZipDatasetWriter:
-        """Create a ds-crawler writer preconfigured from a modality's index."""
+        """Create a ds-crawler writer preconfigured from a modality's index.
+
+        Args:
+            modality_name: Modality whose index metadata should be mirrored.
+            root: Output directory or, when ``zip=True``, output archive.
+            zip: Create a :class:`ds_crawler.ZipDatasetWriter`.
+            metadata_scope: Optional namespace below ``.ds_crawler``. Scoped
+                writers also update ``.ds_crawler/scopes.json``.
+        """
         index_output = self.get_modality_index(modality_name)
         return create_dataset_writer_from_index(
             index_output=index_output,
             root=root,
             zip=zip,
+            metadata_scope=metadata_scope,
         )
 
     def get_writer(self, modality_name: str) -> Callable[..., Any]:

{euler_loading-2.18.0 → euler_loading-2.19.0}/euler_loading/loaders/cpu/generic.py

@@ -11,6 +11,8 @@ Supported file extensions:
 Return types
 ------------
 - **map_2d** / **scattering_coefficient** -- ``np.ndarray`` of shape ``(H, W)`` float32.
+- **semantic_segmentation** / **instance_segmentation** -- integer
+  ``np.ndarray`` of shape ``(H, W)``.
 - **map_3d** / **atmospheric_light** -- ``np.ndarray`` of shape ``(H, W, C)`` float32.
 - **points_3d** -- ``np.ndarray`` of shape ``(3, H, W)`` float32.
 - **spherical_map** -- ``np.ndarray`` of shape ``(H, W, C)`` float32.
@@ -71,6 +73,20 @@ def _load_numpy(path: Union[str, BinaryIO]) -> np.ndarray:
     return arr.astype(np.float32)
 
 
+def _load_label_map(path: Union[str, BinaryIO], dtype: np.dtype) -> np.ndarray:
+    """Load a standardized HW integer label map without converting to float."""
+    ext = os.path.splitext(_get_name(path))[1].lower()
+    if ext == _NPZ_EXTENSION:
+        with np.load(path) as npz:
+            arr = next(iter(npz.values()))
+    else:
+        arr = np.load(path)
+    arr = np.asarray(arr)
+    if arr.ndim != 2:
+        raise ValueError(f"segmentation map must have shape (H, W), got {arr.shape}")
+    return arr.astype(dtype, copy=False)
+
+
 def _write_numpy(path: Union[str, BinaryIO], value: Any) -> None:
     """Write an array to ``.npy`` or ``.npz`` based on extension."""
     ensure_parent(path)
@@ -116,6 +132,28 @@ def map_2d(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None, *, at
     return _load_numpy(path)
 
 
+@modality_meta(
+    modality_type="semantic_segmentation",
+    dtype="uint8",
+    shape="HW",
+    file_formats=[".npy", ".npz"],
+)
+def semantic_segmentation(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None, *, attributes: dict[str, Any] | None = None) -> np.ndarray:
+    """Load an HW uint8 class-id map; 255 is the standardized void id."""
+    return _load_label_map(path, np.dtype(np.uint8))
+
+
+@modality_meta(
+    modality_type="instance_segmentation",
+    dtype="uint16",
+    shape="HW",
+    file_formats=[".npy", ".npz"],
+)
+def instance_segmentation(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None, *, attributes: dict[str, Any] | None = None) -> np.ndarray:
+    """Load an HW uint16 instance-id map; 0 denotes stuff or void."""
+    return _load_label_map(path, np.dtype(np.uint16))
+
+
 @modality_meta(
     modality_type="map_3d",
     dtype="float32",
@@ -244,6 +282,30 @@ def write_map_2d(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] |
     _write_numpy(path, arr)
 
 
+def _write_segmentation(path: Union[str, BinaryIO], value: Any, dtype: np.dtype, name: str) -> None:
+    arr = to_hw(value, name=name).astype(dtype, copy=False)
+    ensure_parent(path)
+    ext = os.path.splitext(_get_name(path))[1].lower()
+    if ext == _NPY_EXTENSION:
+        np.save(path, arr)
+    elif ext == _NPZ_EXTENSION:
+        np.savez_compressed(path, data=arr)
+    else:
+        raise ValueError(f"Unsupported output extension: {ext}")
+
+
+@mark_stream_supported
+def write_semantic_segmentation(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
+    """Write the standardized HW uint8 semantic map (255 means void)."""
+    _write_segmentation(path, value, np.dtype(np.uint8), "semantic_segmentation")
+
+
+@mark_stream_supported
+def write_instance_segmentation(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
+    """Write the standardized HW uint16 instance map (0 means stuff/void)."""
+    _write_segmentation(path, value, np.dtype(np.uint16), "instance_segmentation")
+
+
 @mark_stream_supported
 def write_map_3d(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
     """Write a 3D map to NumPy formats based on extension.

{euler_loading-2.18.0 → euler_loading-2.19.0}/euler_loading/loaders/gpu/generic.py

@@ -11,6 +11,8 @@ Supported file extensions:
 Return types
 ------------
 - **map_2d** / **scattering_coefficient** -- ``torch.FloatTensor`` of shape ``(H, W)``.
+- **semantic_segmentation** / **instance_segmentation** -- integer
+  ``torch.Tensor`` of shape ``(H, W)``.
 - **map_3d** / **atmospheric_light** -- ``torch.FloatTensor`` of shape ``(C, H, W)``.
 - **points_3d** -- ``torch.FloatTensor`` of shape ``(3, H, W)``.
 - **spherical_map** -- ``torch.FloatTensor`` of shape ``(C, H, W)``.
@@ -72,6 +74,20 @@ def _load_numpy(path: Union[str, BinaryIO]) -> np.ndarray:
     return arr.astype(np.float32)
 
 
+def _load_label_map(path: Union[str, BinaryIO], dtype: np.dtype) -> np.ndarray:
+    """Load a standardized HW integer label map without converting to float."""
+    ext = os.path.splitext(_get_name(path))[1].lower()
+    if ext == _NPZ_EXTENSION:
+        with np.load(path) as npz:
+            arr = next(iter(npz.values()))
+    else:
+        arr = np.load(path)
+    arr = np.asarray(arr)
+    if arr.ndim != 2:
+        raise ValueError(f"segmentation map must have shape (H, W), got {arr.shape}")
+    return arr.astype(dtype, copy=False)
+
+
 def _write_numpy(path: Union[str, BinaryIO], value: Any) -> None:
     """Write an array to ``.npy`` or ``.npz`` based on extension."""
     ensure_parent(path)
@@ -118,6 +134,28 @@ def map_2d(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None, *, at
     return torch.from_numpy(arr).contiguous()
 
 
+@modality_meta(
+    modality_type="semantic_segmentation",
+    dtype="uint8",
+    shape="HW",
+    file_formats=[".npy", ".npz"],
+)
+def semantic_segmentation(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None, *, attributes: dict[str, Any] | None = None) -> torch.Tensor:
+    """Load an HW uint8 class-id map; 255 is the standardized void id."""
+    return torch.from_numpy(_load_label_map(path, np.dtype(np.uint8))).contiguous()
+
+
+@modality_meta(
+    modality_type="instance_segmentation",
+    dtype="uint16",
+    shape="HW",
+    file_formats=[".npy", ".npz"],
+)
+def instance_segmentation(path: Union[str, BinaryIO], meta: dict[str, Any] | None = None, *, attributes: dict[str, Any] | None = None) -> torch.Tensor:
+    """Load an HW uint16 instance-id map; 0 denotes stuff or void."""
+    return torch.from_numpy(_load_label_map(path, np.dtype(np.uint16))).contiguous()
+
+
 @modality_meta(
     modality_type="map_3d",
     dtype="float32",
@@ -244,6 +282,30 @@ def write_map_2d(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] |
     _write_numpy(path, arr)
 
 
+def _write_segmentation(path: Union[str, BinaryIO], value: Any, dtype: np.dtype, name: str) -> None:
+    arr = to_hw(value, name=name).astype(dtype, copy=False)
+    ensure_parent(path)
+    ext = os.path.splitext(_get_name(path))[1].lower()
+    if ext == _NPY_EXTENSION:
+        np.save(path, arr)
+    elif ext == _NPZ_EXTENSION:
+        np.savez_compressed(path, data=arr)
+    else:
+        raise ValueError(f"Unsupported output extension: {ext}")
+
+
+@mark_stream_supported
+def write_semantic_segmentation(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
+    """Write the standardized HW uint8 semantic map (255 means void)."""
+    _write_segmentation(path, value, np.dtype(np.uint8), "semantic_segmentation")
+
+
+@mark_stream_supported
+def write_instance_segmentation(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
+    """Write the standardized HW uint16 instance map (0 means stuff/void)."""
+    _write_segmentation(path, value, np.dtype(np.uint16), "instance_segmentation")
+
+
 @mark_stream_supported
 def write_map_3d(path: Union[str, BinaryIO], value: Any, meta: dict[str, Any] | None = None) -> None:
     """Write a 3D map to NumPy formats based on extension.

{euler_loading-2.18.0 → euler_loading-2.19.0}/package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "euler-loading-3d",
+  "name": "ds-crawler-writing",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {}

{euler_loading-2.18.0 → euler_loading-2.19.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "euler-loading"
-version = "2.18.0"
+version = "2.19.0"
 description = "Multi-modal PyTorch dataloader using ds-crawler indices"
 requires-python = ">=3.9"
 dependencies = [

{euler_loading-2.18.0 → euler_loading-2.19.0}/tests/test_loaders.py

@@ -1179,6 +1179,8 @@ class TestRDSWriters:
 GENERIC_LOADER_NAMES = [
     "map_2d",
     "map_3d",
+    "semantic_segmentation",
+    "instance_segmentation",
     "points_3d",
     "scattering_coefficient",
     "atmospheric_light",
@@ -1187,6 +1189,8 @@ GENERIC_LOADER_NAMES = [
 GENERIC_WRITER_NAMES = [
     "write_map_2d",
     "write_map_3d",
+    "write_semantic_segmentation",
+    "write_instance_segmentation",
     "write_points_3d",
     "write_scattering_coefficient",
     "write_atmospheric_light",
@@ -1345,6 +1349,43 @@ class TestGenericBackwardCompat:
         assert torch.equal(top, gpu)
 
 
+class TestGenericSegmentation:
+    """Generic segmentation codecs preserve the standardized CUPS format."""
+
+    @pytest.mark.parametrize(
+        ("function", "writer", "dtype", "torch_dtype", "values"),
+        [
+            ("semantic_segmentation", "write_semantic_segmentation", np.uint8, torch.uint8, [[0, 18], [255, 7]]),
+            ("instance_segmentation", "write_instance_segmentation", np.uint16, torch.uint16, [[0, 1], [257, 65535]]),
+        ],
+    )
+    def test_cpu_gpu_npy_roundtrip(self, tmp_path, function, writer, dtype, torch_dtype, values):
+        expected = np.asarray(values, dtype=dtype)
+        path = tmp_path / f"{function}.npy"
+        getattr(cpu_generic, writer)(str(path), expected)
+
+        cpu_value = getattr(cpu_generic, function)(str(path))
+        gpu_value = getattr(gpu_generic, function)(str(path))
+        assert cpu_value.dtype == dtype
+        assert np.array_equal(cpu_value, expected)
+        assert gpu_value.dtype == torch_dtype
+        assert torch.equal(gpu_value, torch.from_numpy(expected))
+
+    def test_writer_accepts_singleton_channel(self, tmp_path):
+        value = np.array([[[0, 1], [2, 255]]], dtype=np.uint8)
+        path = tmp_path / "semantic.npz"
+        cpu_generic.write_semantic_segmentation(str(path), value)
+        loaded = cpu_generic.semantic_segmentation(str(path))
+        assert loaded.shape == (2, 2)
+        assert np.array_equal(loaded, value[0])
+
+    def test_loader_rejects_non_hw_storage(self, tmp_path):
+        path = tmp_path / "semantic.npy"
+        np.save(path, np.zeros((1, 2, 3), dtype=np.uint8))
+        with pytest.raises(ValueError, match=r"must have shape \(H, W\)"):
+            cpu_generic.semantic_segmentation(str(path))
+
+
 # ---------------------------------------------------------------------------
 # Generic map_2d / map_3d loader tests
 # ---------------------------------------------------------------------------

{euler_loading-2.18.0 → euler_loading-2.19.0}/tests/test_writing.py

@@ -240,6 +240,71 @@ class TestWriteSample:
                 output_index = json.load(io.TextIOWrapper(entry, encoding="utf-8"))
         assert output_index["type"] == "txt"
 
+    def test_output_writer_writes_scoped_metadata_and_manifest(self, tmp_path):
+        def writer(path: str, value: Any, meta: dict[str, Any] | None = None) -> None:
+            with open(path, "w", encoding="utf-8") as f:
+                f.write(str(value))
+
+        index = _flat_index("txt", ["f001"])
+        with patch(
+            "euler_loading.dataset.index_dataset_from_path",
+            return_value=index,
+        ):
+            ds = MultiModalDataset(
+                modalities={
+                    "depth": Modality(
+                        "/data/depth", loader=dummy_loader, writer=writer
+                    )
+                }
+            )
+
+        output_writer = ds.create_output_writer(
+            "depth", tmp_path / "out", metadata_scope="depth"
+        )
+        ds.write_sample(0, {"depth": "prediction"}, output_writer)
+        output_writer.save_index()
+
+        scoped_index = tmp_path / "out" / ".ds_crawler" / "depth" / "output.json"
+        manifest_path = tmp_path / "out" / ".ds_crawler" / "scopes.json"
+        assert json.loads(scoped_index.read_text())["type"] == "txt"
+        manifest = json.loads(manifest_path.read_text())
+        assert manifest["scopes"]["depth"]["metadata_dir"] == ".ds_crawler/depth"
+        assert "output.json" in manifest["scopes"]["depth"]["files"]
+
+    def test_zip_output_writer_writes_scoped_metadata_and_manifest(self, tmp_path):
+        def writer(path: str, value: Any, meta: dict[str, Any] | None = None) -> None:
+            with open(path, "w", encoding="utf-8") as f:
+                f.write(str(value))
+
+        index = _flat_index("txt", ["f001"])
+        with patch(
+            "euler_loading.dataset.index_dataset_from_path",
+            return_value=index,
+        ):
+            ds = MultiModalDataset(
+                modalities={
+                    "depth": Modality(
+                        "/data/depth", loader=dummy_loader, writer=writer
+                    )
+                }
+            )
+
+        output_writer = ds.create_output_writer(
+            "depth", tmp_path / "out.zip", zip=True, metadata_scope="depth"
+        )
+        ds.write_sample(0, {"depth": "prediction"}, output_writer)
+        output_writer.save_index()
+
+        with zipfile.ZipFile(tmp_path / "out.zip") as zf:
+            scoped_index = json.loads(
+                zf.read(".ds_crawler/depth/output.json").decode("utf-8")
+            )
+            manifest = json.loads(
+                zf.read(".ds_crawler/scopes.json").decode("utf-8")
+            )
+        assert scoped_index["type"] == "txt"
+        assert manifest["scopes"]["depth"]["metadata_dir"] == ".ds_crawler/depth"
+
     def test_write_sample_records_attributes_on_dataset_writer_entry(self, tmp_path):
         """write_sample(attributes={modality: {...}}) lands on the output entry."""
         def writer(path: str, value: Any, meta: dict[str, Any] | None = None) -> None: