dataeval 0.73.1__py3-none-any.whl → 0.74.1__py3-none-any.whl

dataeval/metrics/stats/hashstats.py

@@ -116,7 +116,7 @@ class HashStatsProcessor(StatsProcessor[HashStatsOutput]):
     }
 
 
-@set_metadata()
+@set_metadata
 def hashstats(
     images: Iterable[ArrayLike],
     bboxes: Iterable[ArrayLike] | None = None,

dataeval/metrics/stats/labelstats.py

@@ -9,11 +9,11 @@ from typing import Any, Iterable, Mapping, TypeVar
 from numpy.typing import ArrayLike
 
 from dataeval.interop import to_numpy
-from dataeval.output import OutputMetadata, set_metadata
+from dataeval.output import Output, set_metadata
 
 
 @dataclass(frozen=True)
-class LabelStatsOutput(OutputMetadata):
+class LabelStatsOutput(Output):
     """
     Output class for :func:`labelstats` stats metric
 
@@ -57,7 +57,7 @@ def sort(d: Mapping[TKey, Any]) -> dict[TKey, Any]:
     return dict(sorted(d.items(), key=lambda x: x[0]))
 
 
-@set_metadata()
+@set_metadata
 def labelstats(
     labels: Iterable[ArrayLike],
 ) -> LabelStatsOutput:

dataeval/metrics/stats/pixelstats.py

@@ -67,7 +67,7 @@ class PixelStatsProcessor(StatsProcessor[PixelStatsOutput]):
     }
 
 
-@set_metadata()
+@set_metadata
 def pixelstats(
     images: Iterable[ArrayLike],
     bboxes: Iterable[ArrayLike] | None = None,

dataeval/metrics/stats/visualstats.py

@@ -74,7 +74,7 @@ class VisualStatsProcessor(StatsProcessor[VisualStatsOutput]):
     }
 
 
-@set_metadata()
+@set_metadata
 def visualstats(
     images: Iterable[ArrayLike],
     bboxes: Iterable[ArrayLike] | None = None,

dataeval/output.py

@@ -4,9 +4,10 @@ __all__ = []
 
 import inspect
 import sys
+from collections.abc import Mapping
 from datetime import datetime, timezone
-from functools import wraps
-from typing import Any, Callable, Iterable, TypeVar
+from functools import partial, wraps
+from typing import Any, Callable, Iterator, TypeVar
 
 import numpy as np
 
@@ -18,7 +19,7 @@ else:
 from dataeval import __version__
 
 
-class OutputMetadata:
+class Output:
     _name: str
     _execution_time: datetime
     _execution_duration: float
@@ -26,6 +27,9 @@ class OutputMetadata:
     _state: dict[str, str]
     _version: str
 
+    def __str__(self) -> str:
+        return f"{self.__class__.__name__}: {str(self.dict())}"
+
     def dict(self) -> dict[str, Any]:
         return {k: v for k, v in self.__dict__.items() if not k.startswith("_")}
 
@@ -33,58 +37,78 @@ class OutputMetadata:
         return {k.removeprefix("_"): v for k, v in self.__dict__.items() if k.startswith("_")}
 
 
+TKey = TypeVar("TKey", str, int, float, set)
+TValue = TypeVar("TValue")
+
+
+class MappingOutput(Mapping[TKey, TValue], Output):
+    __slots__ = ["_data"]
+
+    def __init__(self, data: Mapping[TKey, TValue]):
+        self._data = data
+
+    def __getitem__(self, key: TKey) -> TValue:
+        return self._data.__getitem__(key)
+
+    def __iter__(self) -> Iterator[TKey]:
+        return self._data.__iter__()
+
+    def __len__(self) -> int:
+        return self._data.__len__()
+
+    def dict(self) -> dict[str, TValue]:
+        return {str(k): v for k, v in self._data.items()}
+
+
 P = ParamSpec("P")
-R = TypeVar("R", bound=OutputMetadata)
+R = TypeVar("R", bound=Output)
 
 
-def set_metadata(
-    state_attr: Iterable[str] | None = None,
-) -> Callable[[Callable[P, R]], Callable[P, R]]:
+def set_metadata(fn: Callable[P, R] | None = None, *, state: list[str] | None = None) -> Callable[P, R]:
     """Decorator to stamp OutputMetadata classes with runtime metadata"""
 
-    def decorator(fn: Callable[P, R]) -> Callable[P, R]:
-        @wraps(fn)
-        def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
-            def fmt(v):
-                if np.isscalar(v):
-                    return v
-                if hasattr(v, "shape"):
-                    return f"{v.__class__.__name__}: shape={getattr(v, 'shape')}"
-                if hasattr(v, "__len__"):
-                    return f"{v.__class__.__name__}: len={len(v)}"
-                return f"{v.__class__.__name__}"
-
-            time = datetime.now(timezone.utc)
-            result = fn(*args, **kwargs)
-            duration = (datetime.now(timezone.utc) - time).total_seconds()
-            fn_params = inspect.signature(fn).parameters
-            # set all params with defaults then update params with mapped arguments and explicit keyword args
-            arguments = {k: None if v.default is inspect.Parameter.empty else v.default for k, v in fn_params.items()}
-            arguments.update(zip(fn_params, args))
-            arguments.update(kwargs)
-            arguments = {k: fmt(v) for k, v in arguments.items()}
-            state = (
-                {k: fmt(getattr(args[0], k)) for k in state_attr if "self" in arguments}
-                if "self" in arguments and state_attr
-                else {}
-            )
-            name = (
-                f"{args[0].__class__.__module__}.{args[0].__class__.__name__}.{fn.__name__}"
-                if "self" in arguments
-                else f"{fn.__module__}.{fn.__qualname__}"
-            )
-            metadata = {
-                "_name": name,
-                "_execution_time": time,
-                "_execution_duration": duration,
-                "_arguments": {k: v for k, v in arguments.items() if k != "self"},
-                "_state": state,
-                "_version": __version__,
-            }
-            for k, v in metadata.items():
-                object.__setattr__(result, k, v)
-            return result
-
-        return wrapper
-
-    return decorator
+    if fn is None:
+        return partial(set_metadata, state=state)  # type: ignore
+
+    @wraps(fn)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+        def fmt(v):
+            if np.isscalar(v):
+                return v
+            if hasattr(v, "shape"):
+                return f"{v.__class__.__name__}: shape={getattr(v, 'shape')}"
+            if hasattr(v, "__len__"):
+                return f"{v.__class__.__name__}: len={len(v)}"
+            return f"{v.__class__.__name__}"
+
+        time = datetime.now(timezone.utc)
+        result = fn(*args, **kwargs)
+        duration = (datetime.now(timezone.utc) - time).total_seconds()
+        fn_params = inspect.signature(fn).parameters
+
+        # set all params with defaults then update params with mapped arguments and explicit keyword args
+        arguments = {k: None if v.default is inspect.Parameter.empty else v.default for k, v in fn_params.items()}
+        arguments.update(zip(fn_params, args))
+        arguments.update(kwargs)
+        arguments = {k: fmt(v) for k, v in arguments.items()}
+        state_attrs = (
+            {k: fmt(getattr(args[0], k)) for k in state if "self" in arguments} if "self" in arguments and state else {}
+        )
+        name = (
+            f"{args[0].__class__.__module__}.{args[0].__class__.__name__}.{fn.__name__}"
+            if "self" in arguments
+            else f"{fn.__module__}.{fn.__qualname__}"
+        )
+        metadata = {
+            "_name": name,
+            "_execution_time": time,
+            "_execution_duration": duration,
+            "_arguments": {k: v for k, v in arguments.items() if k != "self"},
+            "_state": state_attrs,
+            "_version": __version__,
+        }
+        for k, v in metadata.items():
+            object.__setattr__(result, k, v)
+        return result
+
+    return wrapper

dataeval/utils/__init__.py

@@ -4,7 +4,7 @@ in setting up architectures that are guaranteed to work with applicable DataEval
 metrics. Currently DataEval supports both :term:`TensorFlow` and PyTorch backends.
 """
 
-from dataeval import _IS_TENSORFLOW_AVAILABLE, _IS_TORCH_AVAILABLE
+from dataeval import _IS_TORCH_AVAILABLE
 from dataeval.utils.metadata import merge_metadata
 from dataeval.utils.split_dataset import split_dataset
 
@@ -15,10 +15,4 @@ if _IS_TORCH_AVAILABLE:
 
     __all__ += ["torch"]
 
-if _IS_TENSORFLOW_AVAILABLE:
-    from dataeval.utils import tensorflow
-
-    __all__ += ["tensorflow"]
-
-del _IS_TENSORFLOW_AVAILABLE
 del _IS_TORCH_AVAILABLE

dataeval/utils/gmm.py

@@ -0,0 +1,26 @@
+from dataclasses import dataclass
+from typing import Generic, TypeVar
+
+TGMMData = TypeVar("TGMMData")
+
+
+@dataclass
+class GaussianMixtureModelParams(Generic[TGMMData]):
+    """
+    phi : TGMMData
+        Mixture component distribution weights.
+    mu : TGMMData
+        Mixture means.
+    cov : TGMMData
+        Mixture covariance.
+    L : TGMMData
+        Cholesky decomposition of `cov`.
+    log_det_cov : TGMMData
+        Log of the determinant of `cov`.
+    """
+
+    phi: TGMMData
+    mu: TGMMData
+    cov: TGMMData
+    L: TGMMData
+    log_det_cov: TGMMData

dataeval/utils/metadata.py

@@ -131,7 +131,9 @@ def _flatten_dict_inner(
     return items, size
 
 
-def _flatten_dict(d: Mapping[str, Any], sep: str, ignore_lists: bool, fully_qualified: bool) -> dict[str, Any]:
+def _flatten_dict(
+    d: Mapping[str, Any], sep: str, ignore_lists: bool, fully_qualified: bool
+) -> tuple[dict[str, Any], int]:
     """
     Flattens a dictionary and converts values to numeric values when possible.
 
@@ -165,7 +167,7 @@ def _flatten_dict(d: Mapping[str, Any], sep: str, ignore_lists: bool, fully_qual
             output[k] = cv
         elif not isinstance(cv, list):
             output[k] = cv if not size else [cv] * size
-    return output
+    return output, size if size is not None else 1
 
 
 def _is_metadata_dict_of_dicts(metadata: Mapping) -> bool:
@@ -188,7 +190,7 @@ def merge_metadata(
     ignore_lists: bool = False,
     fully_qualified: bool = False,
     as_numpy: bool = False,
-) -> dict[str, list[Any]] | dict[str, NDArray[Any]]:
+) -> tuple[dict[str, list[Any]] | dict[str, NDArray[Any]], NDArray[np.int_]]:
     """
     Merges a collection of metadata dictionaries into a single flattened dictionary of keys and values.
 
@@ -208,8 +210,10 @@ def merge_metadata(
 
     Returns
     -------
-    dict[str, list[Any]] | dict[str, NDArray[Any]]
+    dict[str, list[Any]] or dict[str, NDArray[Any]]
         A single dictionary containing the flattened data as lists or NumPy arrays
+    NDArray[np.int_]
+        Array defining where individual images start, helpful when working with object detection metadata
 
     Note
     ----
@@ -217,9 +221,12 @@ def merge_metadata(
 
     Example
     -------
-    >>> list_metadata = [{"common": 1, "target": [{"a": 1, "b": 3}, {"a": 2, "b": 4}], "source": "example"}]
-    >>> merge_metadata(list_metadata)
+    >>> list_metadata = [{"common": 1, "target": [{"a": 1, "b": 3, "c": 5}, {"a": 2, "b": 4}], "source": "example"}]
+    >>> reorganized_metadata, image_indicies = merge_metadata(list_metadata)
+    >>> reorganized_metadata
     {'common': [1, 1], 'a': [1, 2], 'b': [3, 4], 'source': ['example', 'example']}
+    >>> image_indicies
+    array([0])
     """
     merged: dict[str, list[Any]] = {}
     isect: set[str] = set()
@@ -236,8 +243,11 @@ def merge_metadata(
     else:
         dicts = list(metadata)
 
-    for d in dicts:
-        flattened = _flatten_dict(d, sep="_", ignore_lists=ignore_lists, fully_qualified=fully_qualified)
+    image_repeats = np.zeros(len(dicts))
+    for i, d in enumerate(dicts):
+        flattened, image_repeats[i] = _flatten_dict(
+            d, sep="_", ignore_lists=ignore_lists, fully_qualified=fully_qualified
+        )
         isect = isect.intersection(flattened.keys()) if isect else set(flattened.keys())
         union = union.union(flattened.keys())
         for k, v in flattened.items():
@@ -248,6 +258,16 @@ def merge_metadata(
 
     output: dict[str, Any] = {}
 
+    if image_repeats.sum() == image_repeats.size:
+        image_indicies = np.arange(image_repeats.size)
+    else:
+        image_ids = np.arange(image_repeats.size)
+        image_data = np.concatenate(
+            [np.repeat(image_ids[i], image_repeats[i]) for i in range(image_ids.size)], dtype=np.int_
+        )
+        _, image_unsorted = np.unique(image_data, return_index=True)
+        image_indicies = np.sort(image_unsorted)
+
     if keys:
         output["keys"] = np.array(keys) if as_numpy else keys
 
@@ -255,4 +275,4 @@ def merge_metadata(
         cv = _convert_type(merged[k])
         output[k] = np.array(cv) if as_numpy else cv
 
-    return output
+    return output, image_indicies

dataeval/utils/torch/gmm.py

@@ -0,0 +1,98 @@
+"""
+Adapted for Pytorch from:
+
+Source code derived from Alibi-Detect 0.11.4
+https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
+
+Original code Copyright (c) 2023 Seldon Technologies Ltd
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import torch
+
+from dataeval.utils.gmm import GaussianMixtureModelParams
+
+
+def gmm_params(z: torch.Tensor, gamma: torch.Tensor) -> GaussianMixtureModelParams[torch.Tensor]:
+    """
+    Compute parameters of Gaussian Mixture Model.
+
+    Parameters
+    ----------
+    z : torch.Tensor
+        Observations.
+    gamma : torch.Tensor
+        Mixture probabilities to derive mixture distribution weights from.
+
+    Returns
+    -------
+    GaussianMixtureModelParams(phi, mu, cov, L, log_det_cov)
+        The parameters used to calculate energy.
+    """
+
+    # compute gmm parameters phi, mu and cov
+    N = gamma.shape[0]  # nb of samples in batch
+    sum_gamma = torch.sum(gamma, 0)  # K
+    phi = sum_gamma / N  # K
+    # K x D (D = latent_dim)
+    mu = torch.sum(torch.unsqueeze(gamma, -1) * torch.unsqueeze(z, 1), 0) / torch.unsqueeze(sum_gamma, -1)
+    z_mu = torch.unsqueeze(z, 1) - torch.unsqueeze(mu, 0)  # N x K x D
+    z_mu_outer = torch.unsqueeze(z_mu, -1) * torch.unsqueeze(z_mu, -2)  # N x K x D x D
+
+    # K x D x D
+    cov = torch.sum(torch.unsqueeze(torch.unsqueeze(gamma, -1), -1) * z_mu_outer, 0) / torch.unsqueeze(
+        torch.unsqueeze(sum_gamma, -1), -1
+    )
+
+    # cholesky decomposition of covariance and determinant derivation
+    D = cov.shape[1]
+    eps = 1e-6
+    L = torch.linalg.cholesky(cov + torch.eye(D) * eps)  # K x D x D
+    log_det_cov = 2.0 * torch.sum(torch.log(torch.diagonal(L, dim1=-2, dim2=-1)), 1)  # K
+
+    return GaussianMixtureModelParams(phi, mu, cov, L, log_det_cov)
+
+
+def gmm_energy(
+    z: torch.Tensor,
+    params: GaussianMixtureModelParams[torch.Tensor],
+    return_mean: bool = True,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """
+    Compute sample energy from Gaussian Mixture Model.
+
+    Parameters
+    ----------
+    params : GaussianMixtureModelParams
+        The gaussian mixture model parameters.
+    return_mean : bool, default True
+        Take mean across all sample energies in a batch.
+
+    Returns
+    -------
+    sample_energy
+        The sample energy of the GMM.
+    cov_diag
+        The inverse sum of the diagonal components of the covariance matrix.
+    """
+    D = params.cov.shape[1]
+    z_mu = torch.unsqueeze(z, 1) - torch.unsqueeze(params.mu, 0)  # N x K x D
+    z_mu_T = torch.permute(z_mu, dims=[1, 2, 0])  # K x D x N
+    v = torch.linalg.solve_triangular(params.L, z_mu_T, upper=False)  # K x D x D
+
+    # rewrite sample energy in logsumexp format for numerical stability
+    logits = torch.log(torch.unsqueeze(params.phi, -1)) - 0.5 * (
+        torch.sum(torch.square(v), 1) + float(D) * np.log(2.0 * np.pi) + torch.unsqueeze(params.log_det_cov, -1)
+    )  # K x N
+    sample_energy = -torch.logsumexp(logits, 0)  # N
+
+    if return_mean:
+        sample_energy = torch.mean(sample_energy)
+
+    # inverse sum of variances
+    cov_diag = torch.sum(torch.divide(torch.tensor(1), torch.diagonal(params.cov, dim1=-2, dim2=-1)))
+
+    return sample_energy, cov_diag

dataeval/utils/torch/models.py

@@ -2,8 +2,10 @@ from __future__ import annotations
 
 __all__ = ["AriaAutoencoder", "Encoder", "Decoder"]
 
+import math
 from typing import Any
 
+import torch
 import torch.nn as nn
 
 
@@ -136,3 +138,193 @@ class Decoder(nn.Module):
             The reconstructed output tensor.
         """
         return self.decoder(x)
+
+
+class AE(nn.Module):
+    """
+    An autoencoder model with a separate encoder and decoder. Meant to replace the TensorFlow model called AE, which we
+      used as the core of an autoencoder-based OOD detector, i.e. as an argument to OOD_AE().
+
+    Parameters
+    ----------
+    input_shape : tuple[int, int, int]
+        Number of input channels, number of rows, number of columns.() Number of examples per batch will be inferred
+          at runtime.)
+    """
+
+    def __init__(self, input_shape: tuple[int, int, int]) -> None:
+        super().__init__()
+
+        input_dim = math.prod(input_shape)
+
+        # following is lifted from src/dataeval/utils/tensorflow/_internal/utils.py. It makes an odd staircase that is
+        #  basically proportional to the number of numbers in the image to the 0.8 power. '
+        encoding_dim = int(math.pow(2, int(input_dim.bit_length() * 0.8)))
+
+        self.encoder: Encoder_AE = Encoder_AE(input_shape, encoding_dim)
+
+        self.decoder: Decoder_AE = Decoder_AE(input_shape, encoding_dim, self.encoder.post_op_shape)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        """
+        Perform a forward pass through the encoder and decoder.
+
+        Parameters
+        ----------
+        x : torch.Tensor
+            Input tensor
+
+        Returns
+        -------
+        torch.Tensor
+            The reconstructed output tensor.
+        """
+        x = self.encoder(x)
+        x = self.decoder(x)
+        return x
+
+    def encode(self, x: torch.Tensor) -> torch.Tensor:
+        """
+        Encode the input tensor using the encoder.
+
+        Parameters
+        ----------
+        x : torch.Tensor
+            Input tensor
+
+        Returns
+        -------
+        torch.Tensor
+            The encoded representation of the input tensor.
+        """
+        return self.encoder(x)
+
+
+class Encoder_AE(nn.Module):
+    """
+    A simple encoder to be used in an autoencoder model.
+
+    This is the encoder used to replicate AE, which was a TF function. It consists of a CNN followed by a fully
+      connected layer.
+
+    Parameters
+    ----------
+    channels : int
+        Number of input channels
+
+    input_shape : tuple[int, int, int]
+        number of channels, number of rows, number of columns in input images.
+
+    encoding_dim : the size of the 1D array that emerges from the fully connected layer.
+
+    """
+
+    def __init__(
+        self,
+        input_shape: tuple[int, int, int],
+        encoding_dim: int,
+    ) -> None:
+        super().__init__()
+
+        channels = input_shape[0]
+        nc_in, nc_mid, nc_done = 256, 128, 64
+
+        conv_in = nn.Conv2d(channels, nc_in, 2, stride=1, padding=1)
+        conv_mid = nn.Conv2d(nc_in, nc_mid, 2, stride=1, padding=1)
+        conv_done = nn.Conv2d(nc_mid, nc_done, 2, stride=1)
+
+        self.encoding_ops: nn.Sequential = nn.Sequential(
+            conv_in,
+            nn.LeakyReLU(),
+            nn.MaxPool2d(2),
+            conv_mid,
+            nn.LeakyReLU(),
+            nn.MaxPool2d(2),
+            conv_done,
+        )
+
+        ny, nx = input_shape[1:]
+        self.post_op_shape: tuple[int, int, int] = (nc_done, ny // 4 - 1, nx // 4 - 1)
+        self.flatcon: int = math.prod(self.post_op_shape)
+        self.flatten: nn.Sequential = nn.Sequential(
+            nn.Flatten(),
+            nn.Linear(
+                self.flatcon,
+                encoding_dim,
+            ),
+        )
+
+    def forward(self, x: Any) -> Any:
+        """
+        Perform a forward pass through the AE_torch encoder.
+
+        Parameters
+        ----------
+        x : torch.Tensor
+            Input tensor
+
+        Returns
+        -------
+        torch.Tensor
+            The encoded representation of the input tensor.
+        """
+        x = self.encoding_ops(x)
+
+        x = self.flatten(x)
+
+        return x
+
+
+class Decoder_AE(nn.Module):
+    """
+    A simple decoder to be used in an autoencoder model.
+
+    This is the decoder used by the AriaAutoencoder model.
+
+    Parameters
+    ----------
+    channels : int
+        Number of output channels
+    """
+
+    def __init__(
+        self,
+        input_shape: tuple[int, int, int],
+        encoding_dim: int,
+        post_op_shape: tuple[int, int, int],
+    ) -> None:
+        super().__init__()
+
+        self.post_op_shape = post_op_shape
+        self.input_shape = input_shape  # need to store this for use in forward().
+        channels = input_shape[0]
+
+        self.input: nn.Linear = nn.Linear(encoding_dim, math.prod(post_op_shape))
+
+        self.decoder: nn.Sequential = nn.Sequential(
+            nn.ConvTranspose2d(64, 128, 2, stride=1),
+            nn.LeakyReLU(),
+            nn.ConvTranspose2d(128, 256, 2, stride=2),
+            nn.LeakyReLU(),
+            nn.ConvTranspose2d(256, channels, 2, stride=2),
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        """
+        Perform a forward pass through the decoder.
+
+        Parameters
+        ----------
+        x : torch.Tensor
+            The encoded tensor.
+
+        Returns
+        -------
+        torch.Tensor
+            The reconstructed output tensor.
+        """
+        x = self.input(x)
+        x = x.reshape((-1, *self.post_op_shape))
+        x = self.decoder(x)
+        x = x.reshape((-1, *self.input_shape))
+        return x