evaluators 2.0.0__tar.gz → 2.1.1__tar.gz

{evaluators-2.0.0 → evaluators-2.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: evaluators
-Version: 2.0.0
+Version: 2.1.1
 Summary: Various scene understanding and perception evaluation metrics.
 Author: Kurt Stolle
 Author-email: Kurt Stolle <kurt@computer.org>
@@ -27,7 +27,10 @@ Requires-Dist: lmdb
 Requires-Dist: tifffile
 Requires-Dist: pydantic
 Requires-Dist: opencv-python>=4.13.0.92
+Requires-Dist: imagecodecs>=2026.3.6
+Requires-Dist: lightning>=2.6.1 ; extra == 'lightning'
 Requires-Python: >=3.13, <3.14
+Provides-Extra: lightning
 Description-Content-Type: text/markdown
 
 # Scalable Distributed Evaluation for Computer Vision

{evaluators-2.0.0 → evaluators-2.1.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "evaluators"
-version = "2.0.0"
+version = "2.1.1"
 description = "Various scene understanding and perception evaluation metrics."
 readme = "README.md"
 requires-python = ">=3.13, <3.14"
@@ -35,8 +35,12 @@ dependencies = [
   "tifffile",
   "pydantic",
   "opencv-python>=4.13.0.92",
+  "imagecodecs>=2026.3.6",
 ]
 
+[project.optional-dependencies]
+lightning = ["lightning>=2.6.1"]
+
 [dependency-groups]
 dev = [
   "hypothesis>=6.140.2",

evaluators-2.0.0/sources/evaluators/integrations/pl.py → evaluators-2.1.1/sources/evaluators/integrations/lightning.py

@@ -13,12 +13,9 @@ Exports
 import logging
 import shutil
 from pathlib import Path
-from typing import Type
 
 from evaluators import MetricStream, run_offline_evaluation
 from evaluators.memory import (
-    EncodedMemmapTemporalWriter,
-    LMDBTemporalWriter,
     MemmapTemporalWriter,
     TemporalWriterProtocol,
 )
@@ -39,34 +36,33 @@ class _DistributedWriterBase(Callback):
         self,
         output_dir: str | Path,
         *,
-        chunk_size_frames: int = 1_000,
-        backend: Type[TemporalWriterProtocol] = MemmapTemporalWriter,
+        backend: type[TemporalWriterProtocol] = MemmapTemporalWriter,
         **backend_kwargs,
     ) -> None:
         super().__init__()
         self.output_dir = Path(output_dir)
-        self.chunk_size_frames = chunk_size_frames
         self.backend = backend
         self.backend_kwargs = backend_kwargs
-        # Map: stream_name -> (MetricStream, TemporalWriterProtocol)
-        self._active_streams: dict[str, tuple[MetricStream, TemporalWriterProtocol]] = {}
+        # Map: stream_name -> (MetricStream, Backend)
+        self._active_streams: dict[
+            str, tuple[MetricStream, TemporalWriterProtocol]
+        ] = {}
 
     def _setup_streams(self, pl_module: LightningModule) -> None:
         """Discovers MetricStreams and binds them to dedicated writers."""
         self._active_streams = {}
 
-        discovered_streams: list[MetricStream] = []
-        
         # Robust Discovery: Walk the module tree using standard PyTorch traversal
-        for module in pl_module.modules():
-            if isinstance(module, MetricStream):
-                discovered_streams.append(module)
+        discovered_streams: list[MetricStream] = [
+            module for module in pl_module.modules() if isinstance(module, MetricStream)
+        ]
 
         if not discovered_streams:
             if comms.get_rank() == 0:
                 logger.warning(
-                    f"No MetricStreams found in {pl_module.__class__.__name__}. "
-                    f"{self.__class__.__name__} will be idle."
+                    "No MetricStreams found in %s; %s will be idle.",
+                    pl_module.__class__.__name__,
+                    self.__class__.__name__,
                 )
             return
 
@@ -75,22 +71,22 @@ class _DistributedWriterBase(Callback):
             stream_dir = self.output_dir / stream.name
 
             if stream.schema is None:
-                raise RuntimeError(
+                msg = (
                     f"MetricStream '{stream.name}' has no schema defined. "
                     "Explicit schemas are required for storage backends."
                 )
+                raise RuntimeError(msg)
 
             writer = self.backend(
                 output_dir=str(stream_dir),
                 schema=stream.schema,
-                chunk_size_frames=self.chunk_size_frames,
                 **self.backend_kwargs,
             )
             stream.bind(writer)
             self._active_streams[stream.name] = (stream, writer)
 
     def _close_writers(self) -> None:
-        """Closes all writers and clears the active streams list."""
+        """Close all writers and clears the active streams list."""
         for name, (_, writer) in self._active_streams.items():
             writer.close()
             logger.debug("Closed writer for stream: %s", name)
@@ -129,7 +125,7 @@ class DistributedEvaluationCallback(_DistributedWriterBase):
         output_dir: str | Path,
         *,
         chunk_size_frames: int = 1_000,
-        backend: Type[TemporalWriterProtocol] = MemmapTemporalWriter,
+        backend: type[TemporalWriterProtocol] = MemmapTemporalWriter,
         keep_artifacts: bool = True,
         **backend_kwargs,
     ) -> None:
@@ -180,7 +176,7 @@ class DistributedEvaluationCallback(_DistributedWriterBase):
         comms.barrier()
         if not self.keep_artifacts and comms.get_rank() == 0:
             shutil.rmtree(self.output_dir)
-        
+
         self._active_streams = {}
 
 
@@ -193,6 +189,8 @@ class DistributedInferenceWriter(_DistributedWriterBase):
     def on_predict_start(self, trainer: Trainer, pl_module: LightningModule) -> None:
         self._setup_streams(pl_module)
 
-    def on_predict_epoch_end(self, trainer: Trainer, pl_module: LightningModule) -> None:
+    def on_predict_epoch_end(
+        self, trainer: Trainer, pl_module: LightningModule
+    ) -> None:
         self._close_writers()
         self._active_streams = {}

evaluators-2.1.1/sources/evaluators/memory/utils/encoding.py

@@ -0,0 +1,148 @@
+"""
+Serialization Layer for LMDB Backend.
+=====================================
+
+Provides functional registries to convert PyTorch tensors to bytes (and back)
+using image-specific codecs (PNG, TIFF) and generic raw formats.
+"""
+
+import functools
+
+import io
+from typing import Callable, Literal
+
+import numpy as np
+import tifffile
+import torch
+import torchvision.io
+
+__all__ = [
+    "decode_png",
+    "decode_raw",
+    "decode_tiff",
+    "encode_png",
+    "encode_raw",
+    "encode_tiff",
+]
+
+
+def encode_png(tensor: torch.Tensor) -> bytes:
+    """
+    Encode a tensor to PNG.
+
+    Expects (C, H, W) shape and uint8 dtype.
+    """
+    if tensor.ndim != 3:
+        msg = (
+            f"Expected 3D tensor (C, H, W) for PNG encoding, got shape: {tensor.shape}"
+        )
+        raise ValueError(msg)
+    if tensor.dtype != torch.uint8:
+        msg = f"Expected uint8 dtype for PNG encoding, got: {tensor.dtype}"
+        raise ValueError(msg)
+
+    encoded = torchvision.io.encode_png(tensor.cpu())
+    # encode_png returns a 1D uint8 tensor. Get bytes.
+    return encoded.numpy().tobytes()
+
+
+def decode_png(data: bytes, device: torch.device | None = None) -> torch.Tensor:
+    """
+    Decode PNG bytes back to a tensor.
+    """
+    # Try decode_image which auto-detects
+    t_1d = torch.frombuffer(bytearray(data), dtype=torch.uint8)
+    tensor = torchvision.io.decode_image(
+        t_1d, mode=torchvision.io.ImageReadMode.UNCHANGED
+    )
+    if device is not None:
+        tensor = tensor.to(device)
+    return tensor
+
+
+def encode_tiff(tensor: torch.Tensor, **kwargs) -> bytes:
+    """
+    Encode a tensor to TIFF format using ``tifffile``.
+
+    Suitable for depth maps.
+    """
+    if tensor.ndim != 3:
+        msg = (
+            f"Expected 3D tensor (C, H, W) for TIFF encoding, got shape: {tensor.shape}"
+        )
+        raise ValueError(msg)
+
+    # Determine `imwrite` parameters based on dtype
+    if len(kwargs) > 0:
+        write = functools.partial(tifffile.imwrite, **kwargs)
+    elif tensor.is_floating_point():
+        # Assume depth maps, use LERC compression with a small error threshold of 1mm
+        write = functools.partial(
+            tifffile.imwrite,
+            compression="lerc",
+            compressionargs={"level": 0.001},
+        )
+    else:
+        # Assume segmentation masks
+        write = functools.partial(
+            tifffile.imwrite, compression="zstd", compressionargs={"level": 3}
+        )
+
+    # Permute to (H, W, C), remove channel if C=1, then move to CPU and convert to numpy
+    arr = tensor.permute(1, 2, 0).squeeze(-1).cpu().numpy()
+
+    # Ensure C-order
+    arr = np.ascontiguousarray(arr)
+
+    # Write to an in-memory buffer and get bytes
+    with io.BytesIO() as buf:
+        write(buf, arr, **kwargs)
+        return buf.getvalue()
+
+
+def decode_tiff(data: bytes, device: torch.device | None = None) -> torch.Tensor:
+    """
+    Decode TIFF bytes back to a tensor.
+
+    Returns a tensor in (C, H, W) format.
+    """
+    with io.BytesIO(data) as buf:
+        arr = tifffile.imread(buf)
+
+    # tifffile returns numpy array, convert to tensor
+    tensor = torch.from_numpy(arr)
+
+    # Convert (H, W) to (H, W, 1) if single-channel
+    if tensor.ndim == 2:
+        tensor.unsqueeze_(-1)
+
+    # Permute back to (C, H, W)
+    tensor = tensor.permute(2, 0, 1)
+
+    # Move to device if specified
+    if device is not None:
+        tensor = tensor.to(device)
+
+    # Ensure contiguous memory for better performance in downstream processing
+    return tensor.contiguous()
+
+
+def encode_raw(tensor: torch.Tensor) -> bytes:
+    """
+    Generic raw serialization using PyTorch's native save (uses zip/pickle).
+    Good for small metadata like sequence_ids or unsupported large tensors.
+    """
+    with io.BytesIO() as buf:
+        torch.save(tensor.cpu(), buf)
+        return buf.getvalue()
+
+
+def decode_raw(data: bytes, device: torch.device | None = None) -> torch.Tensor:
+    """
+    Decode generic raw bytes.
+    """
+    with io.BytesIO(data) as buf:
+        tensor = torch.load(buf, weights_only=True)
+    if device is not None:
+        tensor = tensor.to(device)
+    return tensor

evaluators-2.0.0/sources/evaluators/memory/utils/encoding.py

@@ -1,102 +0,0 @@
-"""
-Serialization Layer for LMDB Backend.
-=====================================
-
-Provides functional registries to convert PyTorch tensors to bytes (and back)
-using image-specific codecs (PNG, TIFF) and generic raw formats.
-"""
-
-import io
-from typing import Callable, Literal
-
-import numpy as np
-import tifffile
-import torch
-import torchvision.io
-
-__all__ = [
-    "encode_png",
-    "decode_png",
-    "encode_tiff",
-    "decode_tiff",
-    "encode_raw",
-    "decode_raw",
-]
-
-
-def encode_png(tensor: torch.Tensor) -> bytes:
-    """
-    Encodes a tensor to PNG.
-    Expects (C, H, W) or (H, W). If uint8 is not the dtype, it casts (assumes categorical mask).
-    """
-    if tensor.ndim == 2:
-        tensor = tensor.unsqueeze(0)
-    if tensor.dtype != torch.uint8:
-        # Warning: For segmentation masks, values > 255 will be clipped if not handled carefully,
-        # but typically panoptic ids or labels are packed, or we only support uint8 for png.
-        # Actually, PNG can support 16-bit. torchvision's encode_png only supports uint8.
-        # If >255, we should probably fall back to raw or use a different encoder.
-        if tensor.max() > 255:
-            return encode_raw(tensor)
-        tensor = tensor.to(torch.uint8)
-
-    encoded = torchvision.io.encode_png(tensor.cpu())
-    # encode_png returns a 1D uint8 tensor. Get bytes.
-    return encoded.numpy().tobytes()
-
-
-def decode_png(data: bytes, device: torch.device | None = None) -> torch.Tensor:
-    """
-    Decodes PNG bytes back to a tensor.
-    """
-    # Try decode_image which auto-detects
-    t_1d = torch.frombuffer(bytearray(data), dtype=torch.uint8)
-    tensor = torchvision.io.decode_image(t_1d, mode=torchvision.io.ImageReadMode.UNCHANGED)
-    if device is not None:
-        tensor = tensor.to(device)
-    return tensor
-
-
-def encode_tiff(tensor: torch.Tensor) -> bytes:
-    """
-    Encodes a tensor to TIFF format using tifffile. Suitable for depth maps.
-    """
-    arr = tensor.cpu().numpy()
-    with io.BytesIO() as buf:
-        tifffile.imwrite(buf, arr)
-        return buf.getvalue()
-
-
-def decode_tiff(data: bytes, device: torch.device | None = None) -> torch.Tensor:
-    """
-    Decodes TIFF bytes back to a tensor.
-    """
-    with io.BytesIO(data) as buf:
-        arr = tifffile.imread(buf)
-    
-    # tifffile returns numpy array, convert to tensor
-    tensor = torch.from_numpy(arr)
-    if device is not None:
-        tensor = tensor.to(device)
-    return tensor
-
-
-def encode_raw(tensor: torch.Tensor) -> bytes:
-    """
-    Generic raw serialization using PyTorch's native save (uses zip/pickle).
-    Good for small metadata like sequence_ids or unsupported large tensors.
-    """
-    with io.BytesIO() as buf:
-        torch.save(tensor.cpu(), buf)
-        return buf.getvalue()
-
-
-def decode_raw(data: bytes, device: torch.device | None = None) -> torch.Tensor:
-    """
-    Decodes generic raw bytes.
-    """
-    with io.BytesIO(data) as buf:
-        tensor = torch.load(buf, weights_only=True)
-    if device is not None:
-        tensor = tensor.to(device)
-    return tensor