dataeval 0.69.4__py3-none-any.whl → 0.70.1__py3-none-any.whl

dataeval/_internal/detectors/duplicates.py

@@ -1,13 +1,12 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import Generic, Iterable, Sequence, TypeVar, cast
+from typing import Generic, Iterable, Sequence, TypeVar
 
 from numpy.typing import ArrayLike
 
 from dataeval._internal.detectors.merged_stats import combine_stats, get_dataset_step_from_idx
-from dataeval._internal.flags import ImageStat
-from dataeval._internal.metrics.stats import StatsOutput, imagestats
+from dataeval._internal.metrics.stats.hashstats import HashStatsOutput, hashstats
 from dataeval._internal.output import OutputMetadata, set_metadata
 
 DuplicateGroup = list[int]
@@ -18,6 +17,8 @@ TIndexCollection = TypeVar("TIndexCollection", DuplicateGroup, DatasetDuplicateG
 @dataclass(frozen=True)
 class DuplicatesOutput(Generic[TIndexCollection], OutputMetadata):
     """
+    Output class for :class:`Duplicates` lint detector
+
     Attributes
     ----------
     exact : list[list[int] | dict[int, list[int]]]
@@ -53,26 +54,23 @@ class Duplicates:
     -------
     Initialize the Duplicates class:
 
-    >>> dups = Duplicates()
+    >>> all_dupes = Duplicates()
+    >>> exact_dupes = Duplicates(only_exact=True)
     """
 
     def __init__(self, only_exact: bool = False):
-        self.stats: StatsOutput
+        self.stats: HashStatsOutput
         self.only_exact = only_exact
 
-    def _get_duplicates(self) -> dict[str, list[list[int]]]:
-        stats_dict = self.stats.dict()
-        if "xxhash" in stats_dict:
-            exact_dict: dict[int, list] = {}
-            for i, value in enumerate(stats_dict["xxhash"]):
-                exact_dict.setdefault(value, []).append(i)
-            exact = [sorted(v) for v in exact_dict.values() if len(v) > 1]
-        else:
-            exact = []
+    def _get_duplicates(self, stats: dict) -> dict[str, list[list[int]]]:
+        exact_dict: dict[int, list] = {}
+        for i, value in enumerate(stats["xxhash"]):
+            exact_dict.setdefault(value, []).append(i)
+        exact = [sorted(v) for v in exact_dict.values() if len(v) > 1]
 
-        if "pchash" in stats_dict and not self.only_exact:
+        if not self.only_exact:
             near_dict: dict[int, list] = {}
-            for i, value in enumerate(stats_dict["pchash"]):
+            for i, value in enumerate(stats["pchash"]):
                 near_dict.setdefault(value, []).append(i)
             near = [sorted(v) for v in near_dict.values() if len(v) > 1 and not any(set(v).issubset(x) for x in exact)]
         else:
@@ -84,14 +82,14 @@ class Duplicates:
         }
 
     @set_metadata("dataeval.detectors", ["only_exact"])
-    def evaluate(self, data: Iterable[ArrayLike] | StatsOutput | Sequence[StatsOutput]) -> DuplicatesOutput:
+    def from_stats(self, hashes: HashStatsOutput | Sequence[HashStatsOutput]) -> DuplicatesOutput:
         """
         Returns duplicate image indices for both exact matches and near matches
 
         Parameters
         ----------
-        data : Iterable[ArrayLike], shape - (N, C, H, W) | StatsOutput | Sequence[StatsOutput]
-            A dataset of images in an ArrayLike format or the output(s) from an imagestats metric analysis
+        data : HashStatsOutput | Sequence[HashStatsOutput]
+            The output(s) from a hashstats analysis
 
         Returns
         -------
@@ -100,39 +98,60 @@ class Duplicates:
 
         See Also
         --------
-        imagestats
+        hashstats
 
         Example
         -------
-        >>> dups.evaluate(images)
-        DuplicatesOutput(exact=[[3, 20], [16, 37]], near=[[3, 20, 22], [12, 18], [13, 36], [14, 31], [17, 27], [19, 38, 47]])
-        """  # noqa: E501
+        >>> exact_dupes.from_stats([hashes1, hashes2])
+        DuplicatesOutput(exact=[{0: [3, 20]}, {0: [16], 1: [12]}], near=[])
+        """
 
-        stats, dataset_steps = combine_stats(data)
+        if isinstance(hashes, HashStatsOutput):
+            return DuplicatesOutput(**self._get_duplicates(hashes.dict()))
 
-        if isinstance(stats, StatsOutput):
-            if not stats.xxhash:
-                raise ValueError("StatsOutput must include xxhash information of the images.")
-            if not self.only_exact and not stats.pchash:
-                raise ValueError("StatsOutput must include pchash information of the images for near matches.")
-            self.stats = stats
-        else:
-            flags = ImageStat.XXHASH | (ImageStat(0) if self.only_exact else ImageStat.PCHASH)
-            self.stats = imagestats(cast(Iterable[ArrayLike], data), flags)
+        if not isinstance(hashes, Sequence):
+            raise TypeError("Invalid stats output type; only use output from hashstats.")
 
-        duplicates = self._get_duplicates()
+        combined, dataset_steps = combine_stats(hashes)
+        duplicates = self._get_duplicates(combined.dict())
 
         # split up results from combined dataset into individual dataset buckets
-        if dataset_steps:
-            dup_list: list[list[int]]
-            for dup_type, dup_list in duplicates.items():
-                dup_list_dict = []
-                for idxs in dup_list:
-                    dup_dict = {}
-                    for idx in idxs:
-                        k, v = get_dataset_step_from_idx(idx, dataset_steps)
-                        dup_dict.setdefault(k, []).append(v)
-                    dup_list_dict.append(dup_dict)
-                duplicates[dup_type] = dup_list_dict
+        for dup_type, dup_list in duplicates.items():
+            dup_list_dict = []
+            for idxs in dup_list:
+                dup_dict = {}
+                for idx in idxs:
+                    k, v = get_dataset_step_from_idx(idx, dataset_steps)
+                    dup_dict.setdefault(k, []).append(v)
+                dup_list_dict.append(dup_dict)
+            duplicates[dup_type] = dup_list_dict
 
         return DuplicatesOutput(**duplicates)
+
+    @set_metadata("dataeval.detectors", ["only_exact"])
+    def evaluate(self, data: Iterable[ArrayLike]) -> DuplicatesOutput:
+        """
+        Returns duplicate image indices for both exact matches and near matches
+
+        Parameters
+        ----------
+        data : Iterable[ArrayLike], shape - (N, C, H, W) | StatsOutput | Sequence[StatsOutput]
+            A dataset of images in an ArrayLike format or the output(s) from a hashstats analysis
+
+        Returns
+        -------
+        DuplicatesOutput
+            List of groups of indices that are exact and near matches
+
+        See Also
+        --------
+        hashstats
+
+        Example
+        -------
+        >>> all_dupes.evaluate(images)
+        DuplicatesOutput(exact=[[3, 20], [16, 37]], near=[[3, 20, 22], [12, 18], [13, 36], [14, 31], [17, 27], [19, 38, 47]])
+        """  # noqa: E501
+        self.stats = hashstats(data)
+        duplicates = self._get_duplicates(self.stats.dict())
+        return DuplicatesOutput(**duplicates)

dataeval/_internal/detectors/merged_stats.py

@@ -1,71 +1,40 @@
 from __future__ import annotations
 
-from typing import Sequence, cast
-from warnings import warn
+from copy import deepcopy
+from typing import Sequence, TypeVar
 
 import numpy as np
 
-from dataeval._internal.metrics.stats import StatsOutput
-from dataeval._internal.output import populate_defaults
+from dataeval._internal.metrics.stats.base import BaseStatsOutput
 
+TStatsOutput = TypeVar("TStatsOutput", bound=BaseStatsOutput)
 
-def add_stats(a: StatsOutput, b: StatsOutput) -> StatsOutput:
-    if not isinstance(a, StatsOutput) or not isinstance(b, StatsOutput):
-        raise TypeError(f"Cannot add object of type {type(a)} and type {type(b)}.")
 
-    a_dict = a.dict()
-    b_dict = b.dict()
-    a_keys = set(a_dict)
-    b_keys = set(b_dict)
+def add_stats(a: TStatsOutput, b: TStatsOutput) -> TStatsOutput:
+    if type(a) is not type(b):
+        raise TypeError(f"Types {type(a)} and {type(b)} cannot be added.")
 
-    missing_keys = a_keys - b_keys
-    if missing_keys:
-        raise ValueError(f"Required keys are missing: {missing_keys}.")
+    sum_dict = deepcopy(a.dict())
 
-    extra_keys = b_keys - a_keys
-    if extra_keys:
-        warn(f"Extraneous keys will be dropped: {extra_keys}.")
+    for k in sum_dict:
+        if isinstance(sum_dict[k], list):
+            sum_dict[k].extend(b.dict()[k])
+        else:
+            sum_dict[k] = np.concatenate((sum_dict[k], b.dict()[k]))
 
-    # perform add of multi-channel stats
-    if "ch_idx_map" in a_dict:
-        for k, v in a_dict.items():
-            if k == "ch_idx_map":
-                offset = sum([len(idxs) for idxs in v.values()])
-                for ch_k, ch_v in b_dict[k].items():
-                    if ch_k not in v:
-                        v[ch_k] = []
-                    a_dict[k][ch_k].extend([idx + offset for idx in ch_v])
-            else:
-                for ch_k in b_dict[k]:
-                    if ch_k not in v:
-                        v[ch_k] = b_dict[k][ch_k]
-                    else:
-                        v[ch_k] = np.concatenate((v[ch_k], b_dict[k][ch_k]), axis=1)
-    else:
-        for k in a_dict:
-            if isinstance(a_dict[k], list):
-                a_dict[k].extend(b_dict[k])
-            else:
-                a_dict[k] = np.concatenate((a_dict[k], b_dict[k]))
+    return type(a)(**sum_dict)
 
-    return StatsOutput(**populate_defaults(a_dict, StatsOutput))
-
-
-def combine_stats(stats) -> tuple[StatsOutput | None, list[int]]:
-    dataset_steps = []
-
-    if isinstance(stats, StatsOutput):
-        return stats, dataset_steps
 
+def combine_stats(stats: Sequence[TStatsOutput]) -> tuple[TStatsOutput, list[int]]:
     output = None
-    if isinstance(stats, Sequence) and isinstance(stats[0], StatsOutput):
-        stats = cast(Sequence[StatsOutput], stats)
-        cur_len = 0
-        for s in stats:
-            output = s if output is None else add_stats(output, s)
-            cur_len += len(s)
-            dataset_steps.append(cur_len)
-
+    dataset_steps = []
+    cur_len = 0
+    for s in stats:
+        output = s if output is None else add_stats(output, s)
+        cur_len += len(s)
+        dataset_steps.append(cur_len)
+    if output is None:
+        raise TypeError("Cannot combine empty sequence of stats.")
     return output, dataset_steps
 
 

dataeval/_internal/detectors/ood/ae.py

@@ -15,10 +15,11 @@ import numpy as np
 import tensorflow as tf
 from numpy.typing import ArrayLike
 
-from dataeval._internal.detectors.ood.base import OODBase, OODScore
-from dataeval._internal.interop import to_numpy
+from dataeval._internal.detectors.ood.base import OODBase, OODScoreOutput
+from dataeval._internal.interop import as_numpy
 from dataeval._internal.models.tensorflow.autoencoder import AE
 from dataeval._internal.models.tensorflow.utils import predict_batch
+from dataeval._internal.output import set_metadata
 
 
 class OOD_AE(OODBase):
@@ -46,10 +47,11 @@ class OOD_AE(OODBase):
     ) -> None:
         if loss_fn is None:
             loss_fn = keras.losses.MeanSquaredError()
-        super().fit(to_numpy(x_ref), threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
+        super().fit(as_numpy(x_ref), threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
 
-    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:
-        self._validate(X := to_numpy(X))
+    @set_metadata("dataeval.detectors")
+    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
+        self._validate(X := as_numpy(X))
 
         # reconstruct instances
         X_recon = predict_batch(X, self.model, batch_size=batch_size)
@@ -62,4 +64,4 @@ class OOD_AE(OODBase):
         sorted_fscore_perc = sorted_fscore[:, -n_score_features:]
         iscore = np.mean(sorted_fscore_perc, axis=1)
 
-        return OODScore(iscore, fscore)
+        return OODScoreOutput(iscore, fscore)

dataeval/_internal/detectors/ood/aegmm.py

@@ -14,12 +14,13 @@ import keras
 import tensorflow as tf
 from numpy.typing import ArrayLike
 
-from dataeval._internal.detectors.ood.base import OODGMMBase, OODScore
+from dataeval._internal.detectors.ood.base import OODGMMBase, OODScoreOutput
 from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.autoencoder import AEGMM
 from dataeval._internal.models.tensorflow.gmm import gmm_energy
 from dataeval._internal.models.tensorflow.losses import LossGMM
 from dataeval._internal.models.tensorflow.utils import predict_batch
+from dataeval._internal.output import set_metadata
 
 
 class OOD_AEGMM(OODGMMBase):
@@ -49,7 +50,8 @@ class OOD_AEGMM(OODGMMBase):
             loss_fn = LossGMM()
         super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
 
-    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:
+    @set_metadata("dataeval.detectors")
+    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
         """
         Compute the out-of-distribution (OOD) score for a given dataset.
 
@@ -63,7 +65,7 @@ class OOD_AEGMM(OODGMMBase):
 
         Returns
         -------
-        OODScore
+        OODScoreOutput
             An object containing the instance-level OOD score.
 
         Note
@@ -73,4 +75,4 @@ class OOD_AEGMM(OODGMMBase):
         self._validate(X := to_numpy(X))
         _, z, _ = predict_batch(X, self.model, batch_size=batch_size)
         energy, _ = gmm_energy(z, self.gmm_params, return_mean=False)
-        return OODScore(energy.numpy())  # type: ignore
+        return OODScoreOutput(energy.numpy())  # type: ignore

dataeval/_internal/detectors/ood/base.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from typing import Callable, Literal, NamedTuple, cast
+from typing import Callable, Literal, cast
 
 import keras
 import numpy as np
@@ -26,6 +26,9 @@ from dataeval._internal.output import OutputMetadata, set_metadata
 @dataclass(frozen=True)
 class OODOutput(OutputMetadata):
     """
+    Output class for predictions from :class:`OOD_AE`, :class:`OOD_AEGMM`, :class:`OOD_LLR`,
+    :class:`OOD_VAE`, and :class:`OOD_VAEGMM` out-of-distribution detectors
+
     Attributes
     ----------
     is_ood : NDArray
@@ -41,9 +44,11 @@ class OODOutput(OutputMetadata):
     feature_score: NDArray[np.float32] | None
 
 
-class OODScore(NamedTuple):
+@dataclass(frozen=True)
+class OODScoreOutput(OutputMetadata):
     """
-    NamedTuple containing the instance and (optionally) feature score.
+    Output class for instance and feature scores from :class:`OOD_AE`, :class:`OOD_AEGMM`,
+    :class:`OOD_LLR`, :class:`OOD_VAE`, and :class:`OOD_VAEGMM` out-of-distribution detectors
 
     Parameters
     ----------
@@ -76,7 +81,7 @@ class OODBase(ABC):
     def __init__(self, model: keras.Model) -> None:
         self.model = model
 
-        self._ref_score: OODScore
+        self._ref_score: OODScoreOutput
         self._threshold_perc: float
         self._data_info: tuple[tuple, type] | None = None
 
@@ -102,7 +107,7 @@ class OODBase(ABC):
         self._validate(X)
 
     @abstractmethod
-    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:
+    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
         """
         Compute the out-of-distribution (OOD) scores for a given dataset.
 
@@ -116,7 +121,7 @@ class OODBase(ABC):
 
         Returns
         -------
-        OODScore
+        OODScoreOutput
             An object containing the instance-level and feature-level OOD scores.
         """
 
@@ -197,7 +202,7 @@ class OODBase(ABC):
         # compute outlier scores
         score = self.score(X, batch_size=batch_size)
         ood_pred = score.get(ood_type) > self._threshold_score(ood_type)
-        return OODOutput(is_ood=ood_pred, **score._asdict())
+        return OODOutput(is_ood=ood_pred, **score.dict())
 
 
 class OODGMMBase(OODBase):

dataeval/_internal/detectors/ood/llr.py

@@ -18,11 +18,12 @@ from keras.layers import Input
 from keras.models import Model
 from numpy.typing import ArrayLike, NDArray
 
-from dataeval._internal.detectors.ood.base import OODBase, OODScore
+from dataeval._internal.detectors.ood.base import OODBase, OODScoreOutput
 from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.pixelcnn import PixelCNN
 from dataeval._internal.models.tensorflow.trainer import trainer
 from dataeval._internal.models.tensorflow.utils import predict_batch
+from dataeval._internal.output import set_metadata
 
 
 def build_model(
@@ -124,7 +125,7 @@ class OOD_LLR(OODBase):
         self.sequential = sequential
         self.log_prob = log_prob
 
-        self._ref_score: OODScore
+        self._ref_score: OODScoreOutput
         self._threshold_perc: float
         self._data_info: tuple[tuple, type] | None = None
 
@@ -279,12 +280,13 @@ class OOD_LLR(OODBase):
         logp_b = logp_fn(self.dist_b, X, return_per_feature=return_per_feature, batch_size=batch_size)
         return logp_s - logp_b
 
+    @set_metadata("dataeval.detectors")
     def score(
         self,
         X: ArrayLike,
         batch_size: int = int(1e10),
-    ) -> OODScore:
+    ) -> OODScoreOutput:
         self._validate(X := to_numpy(X))
         fscore = -self._llr(X, True, batch_size=batch_size)
         iscore = -self._llr(X, False, batch_size=batch_size)
-        return OODScore(iscore, fscore)
+        return OODScoreOutput(iscore, fscore)

dataeval/_internal/detectors/ood/vae.py

@@ -15,11 +15,12 @@ import numpy as np
 import tensorflow as tf
 from numpy.typing import ArrayLike
 
-from dataeval._internal.detectors.ood.base import OODBase, OODScore
+from dataeval._internal.detectors.ood.base import OODBase, OODScoreOutput
 from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.autoencoder import VAE
 from dataeval._internal.models.tensorflow.losses import Elbo
 from dataeval._internal.models.tensorflow.utils import predict_batch
+from dataeval._internal.output import set_metadata
 
 
 class OOD_VAE(OODBase):
@@ -67,7 +68,8 @@ class OOD_VAE(OODBase):
             loss_fn = Elbo(0.05)
         super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
 
-    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:
+    @set_metadata("dataeval.detectors")
+    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
         self._validate(X := to_numpy(X))
 
         # sample reconstructed instances
@@ -86,4 +88,4 @@ class OOD_VAE(OODBase):
         sorted_fscore_perc = sorted_fscore[:, -n_score_features:]
         iscore = np.mean(sorted_fscore_perc, axis=1)
 
-        return OODScore(iscore, fscore)
+        return OODScoreOutput(iscore, fscore)

dataeval/_internal/detectors/ood/vaegmm.py

@@ -15,12 +15,13 @@ import numpy as np
 import tensorflow as tf
 from numpy.typing import ArrayLike
 
-from dataeval._internal.detectors.ood.base import OODGMMBase, OODScore
+from dataeval._internal.detectors.ood.base import OODGMMBase, OODScoreOutput
 from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.autoencoder import VAEGMM
 from dataeval._internal.models.tensorflow.gmm import gmm_energy
 from dataeval._internal.models.tensorflow.losses import Elbo, LossGMM
 from dataeval._internal.models.tensorflow.utils import predict_batch
+from dataeval._internal.output import set_metadata
 
 
 class OOD_VAEGMM(OODGMMBase):
@@ -53,7 +54,8 @@ class OOD_VAEGMM(OODGMMBase):
             loss_fn = LossGMM(elbo=Elbo(0.05))
         super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
 
-    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:
+    @set_metadata("dataeval.detectors")
+    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
         """
         Compute the out-of-distribution (OOD) score for a given dataset.
 
@@ -67,7 +69,7 @@ class OOD_VAEGMM(OODGMMBase):
 
         Returns
         -------
-        OODScore
+        OODScoreOutput
             An object containing the instance-level OOD score.
 
         Note
@@ -84,4 +86,4 @@ class OOD_VAEGMM(OODGMMBase):
         energy, _ = gmm_energy(z, self.gmm_params, return_mean=False)
         energy_samples = energy.numpy().reshape((-1, self.samples))  # type: ignore
         iscore = np.mean(energy_samples, axis=-1)
-        return OODScore(iscore)
+        return OODScoreOutput(iscore)