dataeval 0.61.0__py3-none-any.whl

dataeval/_internal/metrics/parity.py

@@ -0,0 +1,180 @@
+import warnings
+from typing import Optional, Tuple
+
+import numpy as np
+import scipy
+
+
+class Parity:
+    """
+    Class for evaluating statistics of observed and expected class labels, including:
+
+    - Chi Squared test for statistical independence between expected and observed labels
+
+    Parameters
+    ----------
+    expected_labels : np.ndarray
+        List of class labels in the expected dataset
+    observed_labels : np.ndarray
+        List of class labels in the observed dataset
+    num_classes : Optional[int]
+        The number of unique classes in the datasets. If this is not specified, it will
+        be inferred from the set of unique labels in expected_labels and observed_labels
+    """
+
+    def __init__(self, expected_labels: np.ndarray, observed_labels: np.ndarray, num_classes: Optional[int] = None):
+        self.set_labels(expected_labels, observed_labels, num_classes)
+
+    def set_labels(self, expected_labels: np.ndarray, observed_labels: np.ndarray, num_classes: Optional[int] = None):
+        """
+        Calculates the label distributions for expected and observed labels
+        and performs validation on the results.
+
+        Parameters
+        ----------
+        expected_labels : np.ndarray
+            List of class labels in the expected dataset
+        observed_labels : np.ndarray
+            List of class labels in the observed dataset
+        num_classes : Optional[int]
+            The number of unique classes in the datasets. If this is not specified, it will
+            be inferred from the set of unique labels in expected_labels and observed_labels
+
+        Raises
+        ------
+        ValueError
+            If x is empty
+        """
+        self.num_classes = num_classes
+
+        # Calculate
+        observed_dist = self._calculate_label_dist(observed_labels)
+        expected_dist = self._calculate_label_dist(expected_labels)
+
+        # Validate
+        self._validate_dist(observed_dist, "observed")
+
+        # Normalize
+        expected_dist = self._normalize_expected_dist(expected_dist, observed_dist)
+
+        # Validate normalized expected distribution
+        self._validate_dist(expected_dist, f"expected for {np.sum(observed_dist)} observations")
+        self._validate_class_balance(expected_dist, observed_dist)
+
+        self._observed_dist = observed_dist
+        self._expected_dist = expected_dist
+
+    def _normalize_expected_dist(self, expected_dist: np.ndarray, observed_dist: np.ndarray) -> np.ndarray:
+        exp_sum = np.sum(expected_dist)
+        obs_sum = np.sum(observed_dist)
+
+        if exp_sum == 0:
+            raise ValueError(
+                f"Expected label distribution {expected_dist} is all zeros. "
+                "Ensure that Parity.expected_dist is set to a list "
+                "with at least one nonzero element"
+            )
+
+        # Renormalize expected distribution to have the same total number of labels as the observed dataset
+        if exp_sum != obs_sum:
+            expected_dist = expected_dist * obs_sum / exp_sum
+
+        return expected_dist
+
+    def _calculate_label_dist(self, labels: np.ndarray) -> np.ndarray:
+        """
+        Calculate the class frequencies associated with a dataset
+
+        Parameters
+        ----------
+        labels : np.ndarray
+            List of class labels in a dataset
+
+        Returns
+        -------
+        label_dist : np.ndarray
+            Array representing label distributions
+        """
+        label_dist = np.bincount(labels, minlength=(self.num_classes if self.num_classes else 0))
+        return label_dist
+
+    def _validate_class_balance(self, expected_dist: np.ndarray, observed_dist: np.ndarray):
+        """
+        Check if the numbers of unique classes in the datasets are unequal
+
+        Parameters
+        ----------
+        expected_dist : np.ndarray
+            Array representing expected label distributions
+        observed_dist : np.ndarray
+            Array representing observed label distributions
+
+        Raises
+        ------
+        ValueError
+            When exp_ld and obs_ld do not have the same number of classes
+        """
+        exp_n_cls = len(expected_dist)
+        obs_n_cls = len(observed_dist)
+        if exp_n_cls != obs_n_cls:
+            raise ValueError(
+                f"Found {obs_n_cls} unique classes in observed label distribution, "
+                f"but found {exp_n_cls} unique classes in expected label distribution,"
+                "This can happen when some class ids have zero instances in one dataset but "
+                "not in the other. When initializing Parity, "
+                "try setting the num_classes parameter to the known number of unique class ids, "
+                "so that classes with zero instances are still included in the distributions."
+            )
+
+    def _validate_dist(self, label_dist: np.ndarray, label_name: str):
+        """
+        Verifies that the given label distribution has labels and checks if
+        any labels have frequencies less than 5.
+
+        Parameters
+        ----------
+        label_dist : np.ndarray
+            Array representing label distributions
+
+        Raises
+        ------
+        ValueError
+            If label_dist is empty
+        Warning
+            If any elements of label_dist are less than 5
+        """
+        if not len(label_dist):
+            raise ValueError(f"No labels found in the {label_name} dataset")
+        if np.any(label_dist < 5):
+            warnings.warn(
+                f"Labels {np.where(label_dist<5)[0]} in {label_name}"
+                " dataset have frequencies less than 5. This may lead"
+                " to invalid chi-squared evaluation."
+            )
+            warnings.warn(
+                f"Labels {np.where(label_dist<5)[0]} in {label_name}"
+                " dataset have frequencies less than 5. This may lead"
+                " to invalid chi-squared evaluation."
+            )
+
+    def evaluate(self) -> Tuple[np.float64, np.float64]:
+        """
+        Perform a one-way chi-squared test between observation frequencies and expected frequencies that
+        tests the null hypothesis that the observed data has the expected frequencies.
+
+        This function acts as an interface to the scipy.stats.chisquare method, which is documented at
+        https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.chisquare.html
+        https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.chisquare.html
+
+        Returns
+        -------
+        np.float64
+            chi-squared value of the test
+        np.float64
+            p-value of the test
+        """
+        cs_result = scipy.stats.chisquare(f_obs=self._observed_dist, f_exp=self._expected_dist)
+
+        chisquared = cs_result.statistic
+        p_value = cs_result.pvalue
+        return chisquared, p_value

dataeval/_internal/metrics/stats.py

@@ -0,0 +1,332 @@
+from enum import Flag
+from typing import Any, Callable, Dict, Generic, Iterable, List, Optional, Sequence, TypeVar, Union
+
+import numpy as np
+from scipy.stats import entropy, kurtosis, skew
+
+from dataeval._internal.flags import ImageHash, ImageProperty, ImageStatistics, ImageStatsFlags, ImageVisuals
+from dataeval._internal.metrics.base import MetricMixin
+from dataeval._internal.metrics.hash import pchash, xxhash
+from dataeval._internal.metrics.utils import edge_filter, get_bitdepth, normalize_image_shape, rescale
+
+QUARTILES = (0, 25, 50, 75, 100)
+
+TBatch = TypeVar("TBatch", bound=Sequence)
+TFlag = TypeVar("TFlag", bound=Flag)
+
+
+class BaseStatsMetric(MetricMixin, Generic[TBatch, TFlag]):
+    def __init__(self, flags: TFlag):
+        self.flags = flags
+        self.results = []
+
+    def update(self, preds: TBatch, targets=None) -> None:
+        """
+        Updates internal metric cache for later calculation
+
+        Parameters
+        ----------
+        batch : Sequence
+            Sequence of images to be processed
+        """
+
+    def compute(self) -> Dict[str, Any]:
+        """
+        Computes the specified measures on the cached values
+
+        Returns
+        -------
+        Dict[str, Any]
+            Dictionary results of the specified measures
+        """
+        return {stat: [result[stat] for result in self.results] for stat in self.results[0]}
+
+    def reset(self) -> None:
+        """
+        Resets the internal metric cache
+        """
+        self.results = []
+
+    def _map(self, func_map: Dict[Flag, Callable]) -> Dict[str, Any]:
+        """Calculates the measures for each flag if it is selected."""
+        results = {}
+        for flag, func in func_map.items():
+            if not flag.name:
+                raise ValueError("Provided flag to set value does not have a name.")
+            if flag & self.flags:
+                results[flag.name.lower()] = func()
+        return results
+
+    def _keys(self) -> List[str]:
+        """Returns the list of measures to be calculated."""
+        flags = (
+            self.flags
+            if isinstance(self.flags, Iterable)  # py3.11
+            else [flag for flag in list(self.flags.__class__) if flag & self.flags]
+        )
+        return [flag.name.lower() for flag in flags if flag.name is not None]
+
+
+class ImageHashMetric(BaseStatsMetric):
+    """
+    Hashes images using the specified algorithms
+
+    Parameters
+    ----------
+    flags : ImageHash
+        Algorithm(s) to calculate a hash as hex digest
+    """
+
+    def __init__(self, flags: ImageHash = ImageHash.ALL):
+        super().__init__(flags)
+
+    def update(self, preds: Iterable[np.ndarray], targets=None) -> None:
+        for data in preds:
+            results = self._map(
+                {
+                    ImageHash.XXHASH: lambda: xxhash(data),
+                    ImageHash.PCHASH: lambda: pchash(data),
+                }
+            )
+            self.results.append(results)
+
+
+class ImagePropertyMetric(BaseStatsMetric):
+    """
+    Calculates specified image properties
+
+    Parameters
+    ----------
+    flags: ImageProperty
+        Property(ies) to calculate for each image
+    """
+
+    def __init__(self, flags: ImageProperty = ImageProperty.ALL):
+        super().__init__(flags)
+
+    def update(self, preds: Iterable[np.ndarray], targets=None) -> None:
+        for data in preds:
+            results = self._map(
+                {
+                    ImageProperty.WIDTH: lambda: np.int32(data.shape[-1]),
+                    ImageProperty.HEIGHT: lambda: np.int32(data.shape[-2]),
+                    ImageProperty.SIZE: lambda: np.int32(data.shape[-1] * data.shape[-2]),
+                    ImageProperty.ASPECT_RATIO: lambda: data.shape[-1] / np.int32(data.shape[-2]),
+                    ImageProperty.CHANNELS: lambda: data.shape[-3],
+                    ImageProperty.DEPTH: lambda: get_bitdepth(data).depth,
+                }
+            )
+            self.results.append(results)
+
+
+class ImageVisualsMetric(BaseStatsMetric):
+    """
+    Calculates specified visual image properties
+
+    Parameters
+    ----------
+    flags: ImageVisuals
+        Property(ies) to calculate for each image
+    """
+
+    def __init__(self, flags: ImageVisuals = ImageVisuals.ALL):
+        super().__init__(flags)
+
+    def update(self, preds: Iterable[np.ndarray], targets=None) -> None:
+        for data in preds:
+            results = self._map(
+                {
+                    ImageVisuals.BRIGHTNESS: lambda: np.mean(rescale(data)),
+                    ImageVisuals.BLURRINESS: lambda: np.std(edge_filter(np.mean(data, axis=0))),
+                    ImageVisuals.MISSING: lambda: np.sum(np.isnan(data)),
+                    ImageVisuals.ZERO: lambda: np.int32(np.count_nonzero(data == 0)),
+                }
+            )
+            self.results.append(results)
+
+
+class ImageStatisticsMetric(BaseStatsMetric):
+    """
+    Calculates descriptive statistics for each image
+
+    Parameters
+    ----------
+    flags: ImageStatistics
+        Statistic(s) to calculate for each image
+    """
+
+    def __init__(self, flags: ImageStatistics = ImageStatistics.ALL):
+        super().__init__(flags)
+
+    def update(self, preds: Iterable[np.ndarray], targets=None) -> None:
+        for data in preds:
+            scaled = rescale(data)
+            if (ImageStatistics.HISTOGRAM | ImageStatistics.ENTROPY) & self.flags:
+                hist = np.histogram(scaled, bins=256, range=(0, 1))[0]
+
+            results = self._map(
+                {
+                    ImageStatistics.MEAN: lambda: np.mean(scaled),
+                    ImageStatistics.STD: lambda: np.std(scaled),
+                    ImageStatistics.VAR: lambda: np.var(scaled),
+                    ImageStatistics.SKEW: lambda: np.float32(skew(scaled.ravel())),
+                    ImageStatistics.KURTOSIS: lambda: np.float32(kurtosis(scaled.ravel())),
+                    ImageStatistics.PERCENTILES: lambda: np.percentile(scaled, q=QUARTILES),
+                    ImageStatistics.HISTOGRAM: lambda: hist,
+                    ImageStatistics.ENTROPY: lambda: np.float32(entropy(hist)),
+                }
+            )
+            self.results.append(results)
+
+
+class ChannelStatisticsMetric(BaseStatsMetric):
+    """
+    Calculates descriptive statistics for each image per channel
+
+    Parameters
+    ----------
+    flags: ImageStatistics
+        Statistic(s) to calculate for each image per channel
+    """
+
+    def __init__(self, flags: ImageStatistics = ImageStatistics.ALL):
+        super().__init__(flags)
+
+    def update(self, preds: Iterable[np.ndarray], targets=None) -> None:
+        for data in preds:
+            scaled = rescale(data)
+            flattened = scaled.reshape(data.shape[0], -1)
+
+            if (ImageStatistics.HISTOGRAM | ImageStatistics.ENTROPY) & self.flags:
+                hist = np.apply_along_axis(lambda x: np.histogram(x, bins=256, range=(0, 1))[0], 1, flattened)
+
+            results = self._map(
+                {
+                    ImageStatistics.MEAN: lambda: np.mean(flattened, axis=1),
+                    ImageStatistics.STD: lambda: np.std(flattened, axis=1),
+                    ImageStatistics.VAR: lambda: np.var(flattened, axis=1),
+                    ImageStatistics.SKEW: lambda: skew(flattened, axis=1),
+                    ImageStatistics.KURTOSIS: lambda: kurtosis(flattened, axis=1),
+                    ImageStatistics.PERCENTILES: lambda: np.percentile(flattened, q=QUARTILES, axis=1).T,
+                    ImageStatistics.HISTOGRAM: lambda: hist,
+                    ImageStatistics.ENTROPY: lambda: entropy(hist, axis=1),
+                }
+            )
+            self.results.append(results)
+
+
+class BaseAggregateMetric(BaseStatsMetric, Generic[TFlag]):
+    FLAG_METRIC_MAP: Dict[type, type]
+    DEFAULT_FLAGS: Sequence[TFlag]
+
+    def __init__(self, flags: Optional[Union[TFlag, Sequence[TFlag]]] = None):
+        flag_dict = {}
+        for flag in flags if isinstance(flags, Sequence) else self.DEFAULT_FLAGS if not flags else [flags]:
+            flag_dict[type(flag)] = flag_dict.setdefault(type(flag), type(flag)(0)) | flag
+        self._metrics_dict = {
+            metric: []
+            for metric in (
+                self.FLAG_METRIC_MAP[flag_class](flag) for flag_class, flag in flag_dict.items() if flag.value != 0
+            )
+        }
+
+
+class ImageStats(BaseAggregateMetric):
+    """
+    Calculates various image property statistics
+
+    Parameters
+    ----------
+    flags: [ImageHash | ImageProperty | ImageStatistics | ImageVisuals], default None
+        Metric(s) to calculate for each image per channel - calculates all metrics if None
+    """
+
+    FLAG_METRIC_MAP = {
+        ImageHash: ImageHashMetric,
+        ImageProperty: ImagePropertyMetric,
+        ImageStatistics: ImageStatisticsMetric,
+        ImageVisuals: ImageVisualsMetric,
+    }
+    DEFAULT_FLAGS = [ImageHash.ALL, ImageProperty.ALL, ImageStatistics.ALL, ImageVisuals.ALL]
+
+    def __init__(self, flags: Optional[Union[ImageStatsFlags, Sequence[ImageStatsFlags]]] = None):
+        super().__init__(flags)
+        self._length = 0
+
+    def update(self, preds: Iterable[np.ndarray], targets=None) -> None:
+        for image in preds:
+            self._length += 1
+            img = normalize_image_shape(image)
+            for metric in self._metrics_dict:
+                metric.update([img])
+
+    def compute(self) -> Dict[str, Any]:
+        for metric in self._metrics_dict:
+            self._metrics_dict[metric] = metric.results
+
+        stats = {}
+        for metric, results in self._metrics_dict.items():
+            for i, result in enumerate(results):
+                for stat in metric._keys():
+                    value = result[stat]
+                    if not isinstance(value, (np.ndarray, np.generic)):
+                        if stat not in stats:
+                            stats[stat] = []
+                        stats[stat].append(result[stat])
+                    else:
+                        if stat not in stats:
+                            shape = () if np.isscalar(result[stat]) else result[stat].shape
+                            stats[stat] = np.empty((self._length,) + shape)
+                        stats[stat][i] = result[stat]
+        return stats
+
+    def reset(self):
+        self._length = 0
+        for metric in self._metrics_dict:
+            metric.reset()
+            self._metrics_dict[metric] = []
+
+
+class ChannelStats(BaseAggregateMetric):
+    FLAG_METRIC_MAP = {ImageStatistics: ChannelStatisticsMetric}
+    DEFAULT_FLAGS = [ImageStatistics.ALL]
+    IDX_MAP = "idx_map"
+
+    def __init__(self, flags: Optional[ImageStatistics] = None) -> None:
+        super().__init__(flags)
+
+    def update(self, preds: Iterable[np.ndarray], targets=None) -> None:
+        for image in preds:
+            img = normalize_image_shape(image)
+            for metric in self._metrics_dict:
+                metric.update([img])
+
+        for metric in self._metrics_dict:
+            self._metrics_dict[metric] = metric.results
+
+    def compute(self) -> Dict[str, Any]:
+        # Aggregate all metrics into a single dictionary
+        stats = {}
+        channel_stats = set()
+        for metric, results in self._metrics_dict.items():
+            for i, result in enumerate(results):
+                for stat in metric._keys():
+                    channel_stats.update(metric._keys())
+                    channels = result[stat].shape[0]
+                    stats.setdefault(self.IDX_MAP, {}).setdefault(channels, {})[i] = None
+                    stats.setdefault(stat, {}).setdefault(channels, []).append(result[stat])
+
+        # Concatenate list of channel statistics numpy
+        for stat in channel_stats:
+            for channel in stats[stat]:
+                stats[stat][channel] = np.array(stats[stat][channel]).T
+
+        for channel in stats[self.IDX_MAP]:
+            stats[self.IDX_MAP][channel] = list(stats[self.IDX_MAP][channel].keys())
+
+        return stats
+
+    def reset(self) -> None:
+        for metric in self._metrics_dict:
+            metric.reset()
+            self._metrics_dict[metric] = []

dataeval/_internal/metrics/uap.py

@@ -0,0 +1,45 @@
+"""
+This module contains the implementation of the
+FR Test Statistic based estimate for the upperbound
+average precision using empirical mean precision
+"""
+
+from typing import Dict
+
+import numpy as np
+from sklearn.metrics import average_precision_score
+
+from dataeval._internal.metrics.base import EvaluateMixin
+
+
+class UAP(EvaluateMixin):
+    """
+    FR Test Statistic based estimate of the empirical mean precision
+
+    Parameters
+    ----------
+    labels : np.ndarray
+        A numpy array of n_samples of class labels with M unique classes.
+
+    scores : np.ndarray
+        A 2D array of class probabilities per image
+    """
+
+    def __init__(self, labels: np.ndarray, scores: np.ndarray) -> None:
+        self.labels = labels
+        self.scores = scores
+
+    def evaluate(self) -> Dict[str, float]:
+        """
+        Returns
+        -------
+        Dict[str, float]
+            uap : The empirical mean precision estimate
+
+        Raises
+        ------
+        ValueError
+            If unique classes M < 2
+        """
+        uap = float(average_precision_score(self.labels, self.scores, average="weighted"))
+        return {"uap": uap}