dataeval 0.64.0__py3-none-any.whl → 0.65.0__py3-none-any.whl

dataeval/_internal/detectors/duplicates.py

@@ -1,9 +1,27 @@
-from typing import Dict, Iterable, List, Literal
+from dataclasses import dataclass
+from typing import Dict, Iterable, List
 
 from numpy.typing import ArrayLike
 
-from dataeval._internal.flags import ImageHash
-from dataeval._internal.metrics.stats import ImageStats
+from dataeval._internal.metrics.stats import StatsOutput
+from dataeval._internal.output import OutputMetadata, set_metadata
+from dataeval.flags import ImageStat
+from dataeval.metrics import imagestats
+
+
+@dataclass(frozen=True)
+class DuplicatesOutput(OutputMetadata):
+    """
+    Attributes
+    ----------
+    exact : List[List[int]]
+        Indices of images that are exact matches
+    near: List[List[int]]
+        Indices of images that are near matches
+    """
+
+    exact: List[List[int]]
+    near: List[List[int]]
 
 
 class Duplicates:
@@ -13,8 +31,8 @@ class Duplicates:
 
     Attributes
     ----------
-    stats : ImageStats(flags=ImageHash.ALL)
-        Base stats class with the flags for checking duplicates
+    stats : StatsOutput
+        Output class of stats
 
     Example
     -------
@@ -23,25 +41,36 @@ class Duplicates:
     >>> dups = Duplicates()
     """
 
-    def __init__(self):
-        self.stats = ImageStats(ImageHash.ALL)
+    def __init__(self, find_exact: bool = True, find_near: bool = True):
+        self.stats: StatsOutput
+        self.find_exact = find_exact
+        self.find_near = find_near
 
-    def _get_duplicates(self) -> dict:
-        exact = {}
-        near = {}
-        for i, value in enumerate(self.results["xxhash"]):
-            exact.setdefault(value, []).append(i)
-        for i, value in enumerate(self.results["pchash"]):
-            near.setdefault(value, []).append(i)
-        exact = [v for v in exact.values() if len(v) > 1]
-        near = [v for v in near.values() if len(v) > 1 and not any(set(v).issubset(x) for x in exact)]
+    def _get_duplicates(self) -> Dict[str, List[List[int]]]:
+        stats_dict = self.stats.dict()
+        if "xxhash" in stats_dict:
+            exact = {}
+            for i, value in enumerate(stats_dict["xxhash"]):
+                exact.setdefault(value, []).append(i)
+            exact = [v for v in exact.values() if len(v) > 1]
+        else:
+            exact = []
+
+        if "pchash" in stats_dict:
+            near = {}
+            for i, value in enumerate(stats_dict["pchash"]):
+                near.setdefault(value, []).append(i)
+            near = [v for v in near.values() if len(v) > 1 and not any(set(v).issubset(x) for x in exact)]
+        else:
+            near = []
 
         return {
             "exact": sorted(exact),
             "near": sorted(near),
         }
 
-    def evaluate(self, images: Iterable[ArrayLike]) -> Dict[Literal["exact", "near"], List[int]]:
+    @set_metadata("dataeval.detectors", ["find_exact", "find_near"])
+    def evaluate(self, images: Iterable[ArrayLike]) -> DuplicatesOutput:
         """
         Returns duplicate image indices for both exact matches and near matches
 
@@ -52,22 +81,19 @@ class Duplicates:
 
         Returns
         -------
-        Dict[str, List[int]]
-            exact :
-                List of groups of indices that are exact matches
-            near :
-                List of groups of indices that are near matches
+        DuplicatesOutput
+            List of groups of indices that are exact and near matches
 
         See Also
         --------
-        ImageStats
+        imagestats
 
         Example
         -------
         >>> dups.evaluate(images)
-        {'exact': [[3, 20], [16, 37]], 'near': [[3, 20, 22], [12, 18], [13, 36], [14, 31], [17, 27], [19, 38, 47]]}
-        """
-        self.stats.reset()
-        self.stats.update(images)
-        self.results = self.stats.compute()
-        return self._get_duplicates()
+        DuplicatesOutput(exact=[[3, 20], [16, 37]], near=[[3, 20, 22], [12, 18], [13, 36], [14, 31], [17, 27], [19, 38, 47]])
+        """  # noqa: E501
+        flag_exact = ImageStat.XXHASH if self.find_exact else ImageStat(0)
+        flag_near = ImageStat.PCHASH if self.find_near else ImageStat(0)
+        self.stats = imagestats(images, flag_exact | flag_near)
+        return DuplicatesOutput(**self._get_duplicates())

dataeval/_internal/detectors/linter.py

@@ -1,15 +1,31 @@
-from typing import Iterable, Literal, Optional, Sequence, Union
+from dataclasses import dataclass
+from typing import Dict, Iterable, Literal, Optional
 
 import numpy as np
-from numpy.typing import ArrayLike
+from numpy.typing import ArrayLike, NDArray
 
-from dataeval._internal.flags import ImageProperty, ImageVisuals, LinterFlags
-from dataeval._internal.metrics.stats import ImageStats
+from dataeval._internal.flags import verify_supported
+from dataeval._internal.output import OutputMetadata, set_metadata
+from dataeval.flags import ImageStat
+from dataeval.metrics import imagestats
+
+
+@dataclass(frozen=True)
+class LinterOutput(OutputMetadata):
+    """
+    Attributes
+    ----------
+    issues : Dict[int, Dict[str, float]]
+        Dictionary containing the indices of outliers and a dictionary showing
+        the issues and calculated values for the given index.
+    """
+
+    issues: Dict[int, Dict[str, float]]
 
 
 def _get_outlier_mask(
-    values: np.ndarray, method: Literal["zscore", "modzscore", "iqr"], threshold: Optional[float]
-) -> np.ndarray:
+    values: NDArray, method: Literal["zscore", "modzscore", "iqr"], threshold: Optional[float]
+) -> NDArray:
     if method == "zscore":
         threshold = threshold if threshold else 3.0
         std = np.std(values)
@@ -18,7 +34,7 @@ def _get_outlier_mask(
     elif method == "modzscore":
         threshold = threshold if threshold else 3.5
         abs_diff = np.abs(values - np.median(values))
-        med_abs_diff = np.median(abs_diff)
+        med_abs_diff = np.median(abs_diff) if np.median(abs_diff) != 0 else np.mean(abs_diff)
         mod_z_score = 0.6745 * abs_diff / med_abs_diff
         return mod_z_score > threshold
     elif method == "iqr":
@@ -36,8 +52,9 @@ class Linter:
 
     Parameters
     ----------
-    flags : [ImageProperty | ImageStatistics | ImageVisuals], default None
+    flags : ImageStat, default ImageStat.ALL_PROPERTIES | ImageStat.ALL_VISUALS
         Metric(s) to calculate for each image - calculates all metrics if None
+        Only supports ImageStat.ALL_STATS
     outlier_method : ["modzscore" | "zscore" | "iqr"], optional - default "modzscore"
         Statistical method used to identify outliers
     outlier_threshold : float, optional - default None
@@ -46,8 +63,8 @@ class Linter:
 
     Attributes
     ----------
-    stats : ImageStats
-        Class to hold the value of each metric for each image
+    stats : Dict[str, Any]
+        Dictionary to hold the value of each metric for each image
 
     See Also
     --------
@@ -81,7 +98,7 @@ class Linter:
 
     Specifying specific metrics to analyze:
 
-    >>> lint = Linter(flags=[ImageProperty.SIZE, ImageVisuals.ALL])
+    >>> lint = Linter(flags=ImageStat.SIZE | ImageStat.ALL_VISUALS)
 
     Specifying an outlier method:
 
@@ -94,19 +111,19 @@ class Linter:
 
     def __init__(
         self,
-        flags: Optional[Union[LinterFlags, Sequence[LinterFlags]]] = None,
+        flags: ImageStat = ImageStat.ALL_PROPERTIES | ImageStat.ALL_VISUALS,
         outlier_method: Literal["zscore", "modzscore", "iqr"] = "modzscore",
         outlier_threshold: Optional[float] = None,
     ):
-        flags = flags if flags is not None else (ImageProperty.ALL, ImageVisuals.ALL)
-        self.stats = ImageStats(flags)
+        verify_supported(flags, ImageStat.ALL_STATS)
+        self.flags = flags
         self.outlier_method: Literal["zscore", "modzscore", "iqr"] = outlier_method
         self.outlier_threshold = outlier_threshold
 
     def _get_outliers(self) -> dict:
         flagged_images = {}
-
-        for stat, values in self.results.items():
+        stats_dict = self.stats.dict()
+        for stat, values in stats_dict.items():
             if not isinstance(values, np.ndarray):
                 continue
 
@@ -118,7 +135,8 @@ class Linter:
 
         return dict(sorted(flagged_images.items()))
 
-    def evaluate(self, images: Iterable[ArrayLike]) -> dict:
+    @set_metadata("dataeval.detectors", ["flags", "outlier_method", "outlier_threshold"])
+    def evaluate(self, images: Iterable[ArrayLike]) -> LinterOutput:
         """
         Returns indices of outliers with the issues identified for each
 
@@ -130,8 +148,8 @@ class Linter:
 
         Returns
         -------
-        Dict[int, Dict[str, float]]
-            Dictionary containing the indices of outliers and a dictionary showing
+        LinterOutput
+            Output class containing the indices of outliers and a dictionary showing
             the issues and calculated values for the given index.
 
         Example
@@ -139,9 +157,7 @@ class Linter:
         Evaluate the dataset:
 
         >>> lint.evaluate(images)
-        {18: {'brightness': 0.78}, 25: {'brightness': 0.98}}
+        LinterOutput(issues={18: {'brightness': 0.78}, 25: {'brightness': 0.98}})
         """
-        self.stats.reset()
-        self.stats.update(images)
-        self.results = self.stats.compute()
-        return self._get_outliers()
+        self.stats = imagestats(images, self.flags)
+        return LinterOutput(self._get_outliers())

dataeval/_internal/detectors/ood/base.py

@@ -7,16 +7,36 @@ Licensed under Apache Software License (Apache 2.0)
 """
 
 from abc import ABC, abstractmethod
-from typing import Callable, Dict, List, Literal, NamedTuple, Optional, Tuple, cast
+from dataclasses import dataclass
+from typing import Callable, List, Literal, NamedTuple, Optional, Tuple, cast
 
 import keras
 import numpy as np
 import tensorflow as tf
-from numpy.typing import ArrayLike
+from numpy.typing import ArrayLike, NDArray
 
 from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.gmm import GaussianMixtureModelParams, gmm_params
 from dataeval._internal.models.tensorflow.trainer import trainer
+from dataeval._internal.output import OutputMetadata, set_metadata
+
+
+@dataclass(frozen=True)
+class OODOutput(OutputMetadata):
+    """
+    Attributes
+    ----------
+    is_ood : NDArray[np.bool_]
+        Array of images that are detected as out of distribution
+    instance_score : NDArray[np.float32]
+        Instance score of the evaluated dataset
+    feature_score : Optional[NDArray[np.float32]]
+        Feature score, if available, of the evaluated dataset
+    """
+
+    is_ood: NDArray[np.bool_]
+    instance_score: NDArray[np.float32]
+    feature_score: Optional[NDArray[np.float32]]
 
 
 class OODScore(NamedTuple):
@@ -25,16 +45,16 @@ class OODScore(NamedTuple):
 
     Parameters
     ----------
-    instance_score : np.ndarray
+    instance_score : NDArray[np.float32]
         Instance score of the evaluated dataset.
-    feature_score : Optional[np.ndarray], default None
+    feature_score : Optional[NDArray[np.float32]], default None
         Feature score, if available, of the evaluated dataset.
     """
 
-    instance_score: np.ndarray
-    feature_score: Optional[np.ndarray] = None
+    instance_score: NDArray[np.float32]
+    feature_score: Optional[NDArray[np.float32]] = None
 
-    def get(self, ood_type: Literal["instance", "feature"]) -> np.ndarray:
+    def get(self, ood_type: Literal["instance", "feature"]) -> NDArray:
         return self.instance_score if ood_type == "instance" or self.feature_score is None else self.feature_score
 
 
@@ -49,18 +69,18 @@ class OODBase(ABC):
         if not isinstance(model, keras.Model):
             raise TypeError("Model should be of type 'keras.Model'.")
 
-    def _get_data_info(self, X: np.ndarray) -> Tuple[tuple, type]:
+    def _get_data_info(self, X: NDArray) -> Tuple[tuple, type]:
         if not isinstance(X, np.ndarray):
-            raise TypeError("Dataset should of type: `np.ndarray`.")
+            raise TypeError("Dataset should of type: `NDArray`.")
         return X.shape[1:], X.dtype.type
 
-    def _validate(self, X: np.ndarray) -> None:
+    def _validate(self, X: NDArray) -> None:
         check_data_info = self._get_data_info(X)
         if self._data_info is not None and check_data_info != self._data_info:
             raise RuntimeError(f"Expect data of type: {self._data_info[1]} and shape: {self._data_info[0]}. \
                                Provided data is type: {check_data_info[1]} and shape: {check_data_info[0]}.")
 
-    def _validate_state(self, X: np.ndarray, additional_attrs: Optional[List[str]] = None) -> None:
+    def _validate_state(self, X: NDArray, additional_attrs: Optional[List[str]] = None) -> None:
         attrs = ["_data_info", "_threshold_perc", "_ref_score"]
         attrs = attrs if additional_attrs is None else attrs + additional_attrs
         if not all(hasattr(self, attr) for attr in attrs) or any(getattr(self, attr) for attr in attrs) is None:
@@ -132,12 +152,13 @@ class OODBase(ABC):
         self._ref_score = self.score(x_ref, batch_size)
         self._threshold_perc = threshold_perc
 
+    @set_metadata("dataeval.detectors")
     def predict(
         self,
         X: ArrayLike,
         batch_size: int = int(1e10),
         ood_type: Literal["feature", "instance"] = "instance",
-    ) -> Dict[str, np.ndarray]:
+    ) -> OODOutput:
         """
         Predict whether instances are out-of-distribution or not.
 
@@ -157,8 +178,8 @@ class OODBase(ABC):
         self._validate_state(X := to_numpy(X))
         # compute outlier scores
         score = self.score(X, batch_size=batch_size)
-        ood_pred = (score.get(ood_type) > self._threshold_score(ood_type)).astype(int)
-        return {**{"is_ood": ood_pred}, **score._asdict()}
+        ood_pred = score.get(ood_type) > self._threshold_score(ood_type)
+        return OODOutput(is_ood=ood_pred, **score._asdict())
 
 
 class OODGMMBase(OODBase):
@@ -166,7 +187,7 @@ class OODGMMBase(OODBase):
         super().__init__(model)
         self.gmm_params: GaussianMixtureModelParams
 
-    def _validate_state(self, X: np.ndarray, additional_attrs: Optional[List[str]] = None) -> None:
+    def _validate_state(self, X: NDArray, additional_attrs: Optional[List[str]] = None) -> None:
         if additional_attrs is None:
             additional_attrs = ["gmm_params"]
         super()._validate_state(X, additional_attrs)

dataeval/_internal/detectors/ood/llr.py

@@ -14,7 +14,7 @@ import numpy as np
 import tensorflow as tf
 from keras.layers import Input
 from keras.models import Model
-from numpy.typing import ArrayLike
+from numpy.typing import ArrayLike, NDArray
 
 from dataeval._internal.detectors.ood.base import OODBase, OODScore
 from dataeval._internal.interop import to_numpy
@@ -52,7 +52,7 @@ def build_model(
 
 
 def mutate_categorical(
-    X: np.ndarray,
+    X: NDArray,
     rate: float,
     seed: int = 0,
     feature_range: tuple = (0, 255),
@@ -221,10 +221,10 @@ class OOD_LLR(OODBase):
     def _logp(
         self,
         dist,
-        X: np.ndarray,
+        X: NDArray,
         return_per_feature: bool = False,
         batch_size: int = int(1e10),
-    ) -> np.ndarray:
+    ) -> NDArray:
         """
         Compute log probability of a batch of instances under the generative model.
         """
@@ -235,10 +235,10 @@ class OOD_LLR(OODBase):
     def _logp_alt(
         self,
         model: keras.Model,
-        X: np.ndarray,
+        X: NDArray,
         return_per_feature: bool = False,
         batch_size: int = int(1e10),
-    ) -> np.ndarray:
+    ) -> NDArray:
         """
         Compute log probability of a batch of instances with the user defined log_prob function.
         """
@@ -254,7 +254,7 @@ class OOD_LLR(OODBase):
             axis = tuple(np.arange(len(logp.shape))[1:])
             return np.mean(logp, axis=axis)
 
-    def _llr(self, X: np.ndarray, return_per_feature: bool, batch_size: int = int(1e10)) -> np.ndarray:
+    def _llr(self, X: NDArray, return_per_feature: bool, batch_size: int = int(1e10)) -> NDArray:
         """
         Compute likelihood ratios.
 

dataeval/_internal/flags.py

@@ -1,37 +1,31 @@
-from enum import Flag, auto
-from typing import Union
+from enum import IntFlag, auto
+from functools import reduce
+from typing import Dict, Iterable, TypeVar, Union, cast
 
+TFlag = TypeVar("TFlag", bound=IntFlag)
 
-class auto_all:
-    def __get__(self, _, cls):
-        return ~cls(0)
 
+class ImageStat(IntFlag):
+    """
+    Flags for calculating image and channel statistics
+    """
 
-class ImageHash(Flag):
+    # HASHES
     XXHASH = auto()
     PCHASH = auto()
-    ALL = auto_all()
-
-
-class ImageProperty(Flag):
+    # PROPERTIES
     WIDTH = auto()
     HEIGHT = auto()
     SIZE = auto()
     ASPECT_RATIO = auto()
     CHANNELS = auto()
     DEPTH = auto()
-    ALL = auto_all()
-
-
-class ImageVisuals(Flag):
+    # VISUALS
     BRIGHTNESS = auto()
     BLURRINESS = auto()
     MISSING = auto()
     ZERO = auto()
-    ALL = auto_all()
-
-
-class ImageStatistics(Flag):
+    # PIXEL STATS
     MEAN = auto()
     STD = auto()
     VAR = auto()
@@ -40,8 +34,35 @@ class ImageStatistics(Flag):
     ENTROPY = auto()
     PERCENTILES = auto()
     HISTOGRAM = auto()
-    ALL = auto_all()
+    # JOINT FLAGS
+    ALL_HASHES = XXHASH | PCHASH
+    ALL_PROPERTIES = WIDTH | HEIGHT | SIZE | ASPECT_RATIO | CHANNELS | DEPTH
+    ALL_VISUALS = BRIGHTNESS | BLURRINESS | MISSING | ZERO
+    ALL_PIXELSTATS = MEAN | STD | VAR | SKEW | KURTOSIS | ENTROPY | PERCENTILES | HISTOGRAM
+    ALL_STATS = ALL_PROPERTIES | ALL_VISUALS | ALL_PIXELSTATS
+    ALL = ALL_HASHES | ALL_STATS
+
+
+def is_distinct(flag: IntFlag) -> bool:
+    return (flag & (flag - 1) == 0) and flag != 0
+
+
+def to_distinct(flag: TFlag) -> Dict[TFlag, str]:
+    """
+    Returns a distinct set of all flags set on the input flag and their names
+
+    NOTE: this is supported natively in Python 3.11, but for earlier versions we need
+    to use a combination of list comprehension and bit fiddling to determine distinct
+    flag values from joint aliases.
+    """
+    if isinstance(flag, Iterable):  # >= py311
+        return {f: f.name.lower() for f in flag if f.name}
+    else:  # < py311
+        return {f: f.name.lower() for f in list(flag.__class__) if f & flag and is_distinct(f) and f.name}
 
 
-ImageStatsFlags = Union[ImageHash, ImageProperty, ImageVisuals, ImageStatistics]
-LinterFlags = Union[ImageProperty, ImageVisuals, ImageStatistics]
+def verify_supported(flag: TFlag, flags: Union[TFlag, Iterable[TFlag]]):
+    supported = flags if isinstance(flags, flag.__class__) else cast(TFlag, reduce(lambda a, b: a | b, flags))  # type: ignore
+    unsupported = flag & ~supported
+    if unsupported:
+        raise ValueError(f"Unsupported flags {unsupported} called.  Only {supported} flags are supported.")

dataeval/_internal/interop.py

@@ -2,7 +2,7 @@ from importlib import import_module
 from typing import Iterable, Optional
 
 import numpy as np
-from numpy.typing import ArrayLike
+from numpy.typing import ArrayLike, NDArray
 
 module_cache = {}
 
@@ -20,7 +20,7 @@ def try_import(module_name):
     return module
 
 
-def to_numpy(array: Optional[ArrayLike]) -> np.ndarray:
+def to_numpy(array: Optional[ArrayLike]) -> NDArray:
     if array is None:
         return np.ndarray([])
 

dataeval/_internal/metrics/balance.py

@@ -1,14 +1,17 @@
 import warnings
-from typing import Dict, List, NamedTuple, Sequence
+from dataclasses import dataclass
+from typing import Dict, List, Sequence
 
 import numpy as np
 from numpy.typing import NDArray
 from sklearn.feature_selection import mutual_info_classif, mutual_info_regression
 
 from dataeval._internal.metrics.utils import entropy, preprocess_metadata
+from dataeval._internal.output import OutputMetadata, set_metadata
 
 
-class BalanceOutput(NamedTuple):
+@dataclass(frozen=True)
+class BalanceOutput(OutputMetadata):
     """
     Attributes
     ----------
@@ -39,6 +42,7 @@ def validate_num_neighbors(num_neighbors: int) -> int:
     return num_neighbors
 
 
+@set_metadata("dataeval.metrics")
 def balance(class_labels: Sequence[int], metadata: List[Dict], num_neighbors: int = 5) -> BalanceOutput:
     """
     Mutual information (MI) between factors (class label, metadata, label/image properties)
@@ -83,6 +87,9 @@ def balance(class_labels: Sequence[int], metadata: List[Dict], num_neighbors: in
         tgt = data[:, idx]
 
         if is_categorical[idx]:
+            if tgt.dtype == float:
+                # map to unique integers if categorical
+                _, tgt = np.unique(tgt, return_inverse=True)
             # categorical target
             mi[idx, :] = mutual_info_classif(
                 data,
@@ -107,6 +114,7 @@ def balance(class_labels: Sequence[int], metadata: List[Dict], num_neighbors: in
     return BalanceOutput(nmi)
 
 
+@set_metadata("dataeval.metrics")
 def balance_classwise(class_labels: Sequence[int], metadata: List[Dict], num_neighbors: int = 5) -> BalanceOutput:
     """
     Compute mutual information (analogous to correlation) between metadata factors

dataeval/_internal/metrics/ber.py

@@ -7,7 +7,8 @@ Learning to Bound the Multi-class Bayes Error (Th. 3 and Th. 4)
 https://arxiv.org/abs/1811.06419
 """
 
-from typing import Literal, NamedTuple, Tuple
+from dataclasses import dataclass
+from typing import Literal, Tuple
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
@@ -16,9 +17,11 @@ from scipy.stats import mode
 
 from dataeval._internal.interop import to_numpy
 from dataeval._internal.metrics.utils import compute_neighbors, get_classes_counts, get_method, minimum_spanning_tree
+from dataeval._internal.output import OutputMetadata, set_metadata
 
 
-class BEROutput(NamedTuple):
+@dataclass(frozen=True)
+class BEROutput(OutputMetadata):
     """
     Attributes
     ----------
@@ -73,9 +76,6 @@ def ber_knn(X: NDArray, y: NDArray, k: int) -> Tuple[float, float]:
         The upper and lower bounds of the bayes error rate
     """
     M, N = get_classes_counts(y)
-
-    # All features belong on second dimension
-    X = X.reshape((X.shape[0], -1))
     nn_indices = compute_neighbors(X, X, k=k)
     nn_indices = np.expand_dims(nn_indices, axis=1) if nn_indices.ndim == 1 else nn_indices
     modal_class = mode(y[nn_indices], axis=1, keepdims=True).mode.squeeze()
@@ -107,6 +107,7 @@ def knn_lowerbound(value: float, classes: int, k: int) -> float:
 BER_FN_MAP = {"KNN": ber_knn, "MST": ber_mst}
 
 
+@set_metadata("dataeval.metrics")
 def ber(images: ArrayLike, labels: ArrayLike, k: int = 1, method: Literal["KNN", "MST"] = "KNN") -> BEROutput:
     """
     An estimator for Multi-class Bayes Error Rate using FR or KNN test statistic basis