dataeval 0.64.0__py3-none-any.whl → 0.65.0__py3-none-any.whl

dataeval/_internal/metrics/stats.py

@@ -1,345 +1,224 @@
-from abc import abstractmethod
-from enum import Flag
-from typing import Any, Callable, Dict, Generic, Iterable, List, Optional, Sequence, TypeVar, Union
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, Iterable, List
 
 import numpy as np
-from numpy.typing import ArrayLike
+from numpy.typing import ArrayLike, NDArray
 from scipy.stats import entropy, kurtosis, skew
 
-from dataeval._internal.flags import ImageHash, ImageProperty, ImageStatistics, ImageStatsFlags, ImageVisuals
+from dataeval._internal.flags import ImageStat, to_distinct, verify_supported
 from dataeval._internal.interop import to_numpy_iter
-from dataeval._internal.metrics.base import EvaluateMixin
 from dataeval._internal.metrics.utils import edge_filter, get_bitdepth, normalize_image_shape, pchash, rescale, xxhash
+from dataeval._internal.output import OutputMetadata, populate_defaults, set_metadata
 
-QUARTILES = (0, 25, 50, 75, 100)
-
-TBatch = TypeVar("TBatch", bound=Sequence[ArrayLike])
-TFlag = TypeVar("TFlag", bound=Flag)
-
-
-class BaseStatsMetric(EvaluateMixin, Generic[TBatch, TFlag]):
-    def __init__(self, flags: TFlag):
-        self.flags = flags
-        self.results = []
-
-    @abstractmethod
-    def update(self, images: TBatch) -> None:
-        """
-        Updates internal metric cache for later calculation
-
-        Parameters
-        ----------
-        batch : Sequence
-            Sequence of images to be processed
-        """
-
-    def compute(self) -> Dict[str, Any]:
-        """
-        Computes the specified measures on the cached values
-
-        Returns
-        -------
-        Dict[str, Any]
-            Dictionary results of the specified measures
-        """
-        return {stat: [result[stat] for result in self.results] for stat in self.results[0]}
-
-    def reset(self) -> None:
-        """
-        Resets the internal metric cache
-        """
-        self.results = []
-
-    def _map(self, func_map: Dict[Flag, Callable]) -> Dict[str, Any]:
-        """Calculates the measures for each flag if it is selected."""
-        results = {}
-        for flag, func in func_map.items():
-            if not flag.name:
-                raise ValueError("Provided flag to set value does not have a name.")
-            if flag & self.flags:
-                results[flag.name.lower()] = func()
-        return results
-
-    def _keys(self) -> List[str]:
-        """Returns the list of measures to be calculated."""
-        flags = (
-            self.flags
-            if isinstance(self.flags, Iterable)  # py3.11
-            else [flag for flag in list(self.flags.__class__) if flag & self.flags]
-        )
-        return [flag.name.lower() for flag in flags if flag.name is not None]
-
-    def evaluate(self, images: TBatch) -> Dict[str, Any]:
-        """Calculate metric results given a single batch of images"""
-        if self.results:
-            raise RuntimeError("Call reset before calling evaluate")
-
-        self.update(images)
-        results = self.compute()
-        self.reset()
-        return results
-
-
-class ImageHashMetric(BaseStatsMetric):
-    """
-    Hashes images using the specified algorithms
-
-    Parameters
-    ----------
-    flags : ImageHash
-        Algorithm(s) to calculate a hash as hex digest
-    """
-
-    def __init__(self, flags: ImageHash = ImageHash.ALL):
-        super().__init__(flags)
-
-    def update(self, images: Iterable[ArrayLike]) -> None:
-        for image in to_numpy_iter(images):
-            results = self._map(
-                {
-                    ImageHash.XXHASH: lambda: xxhash(image),
-                    ImageHash.PCHASH: lambda: pchash(image),
-                }
-            )
-            self.results.append(results)
+CH_IDX_MAP = "ch_idx_map"
 
 
-class ImagePropertyMetric(BaseStatsMetric):
+@dataclass(frozen=True)
+class StatsOutput(OutputMetadata):
     """
-    Calculates specified image properties
-
-    Parameters
+    Attributes
     ----------
-    flags: ImageProperty
-        Property(ies) to calculate for each image
+    xxhash : List[str]
+        xxHash hash of the images as a hex string
+    pchash : List[str]
+        Perception hash of the images as a hex string
+    width: NDArray[np.uint16]
+        Width of the images in pixels
+    height: NDArray[np.uint16]
+        Height of the images in pixels
+    channels: NDArray[np.uint8]
+        Channel count of the images in pixels
+    size: NDArray[np.uint32]
+        Size of the images in pixels
+    aspect_ratio: NDArray[np.float16]
+        Aspect ratio of the images (width/height)
+    depth: NDArray[np.uint8]
+        Color depth of the images in bits
+    brightness: NDArray[np.float16]
+        Brightness of the images
+    blurriness: NDArray[np.float16]
+        Blurriness of the images
+    missing: NDArray[np.float16]
+        Percentage of the images with missing pixels
+    zero: NDArray[np.float16]
+        Percentage of the images with zero value pixels
+    mean: NDArray[np.float16]
+        Mean of the pixel values of the images
+    std: NDArray[np.float16]
+        Standard deviation of the pixel values of the images
+    var: NDArray[np.float16]
+        Variance of the pixel values of the images
+    skew: NDArray[np.float16]
+        Skew of the pixel values of the images
+    kurtosis: NDArray[np.float16]
+        Kurtosis of the pixel values of the images
+    percentiles: NDArray[np.float16]
+        Percentiles of the pixel values of the images with quartiles of (0, 25, 50, 75, 100)
+    histogram: NDArray[np.uint32]
+        Histogram of the pixel values of the images across 256 bins scaled between 0 and 1
+    entropy: NDArray[np.float16]
+        Entropy of the pixel values of the images
+    ch_idx_map: Dict[int, List[int]]
+        Per-channel mapping of indices for each metric
     """
 
-    def __init__(self, flags: ImageProperty = ImageProperty.ALL):
-        super().__init__(flags)
+    xxhash: List[str]
+    pchash: List[str]
+    width: NDArray[np.uint16]
+    height: NDArray[np.uint16]
+    channels: NDArray[np.uint8]
+    size: NDArray[np.uint32]
+    aspect_ratio: NDArray[np.float16]
+    depth: NDArray[np.uint8]
+    brightness: NDArray[np.float16]
+    blurriness: NDArray[np.float16]
+    missing: NDArray[np.float16]
+    zero: NDArray[np.float16]
+    mean: NDArray[np.float16]
+    std: NDArray[np.float16]
+    var: NDArray[np.float16]
+    skew: NDArray[np.float16]
+    kurtosis: NDArray[np.float16]
+    percentiles: NDArray[np.float16]
+    histogram: NDArray[np.uint32]
+    entropy: NDArray[np.float16]
+    ch_idx_map: Dict[int, List[int]]
+
+    def dict(self):
+        return {k: v for k, v in self.__dict__.items() if not k.startswith("_") and len(v) > 0}
 
-    def update(self, images: Iterable[ArrayLike]) -> None:
-        for image in to_numpy_iter(images):
-            results = self._map(
-                {
-                    ImageProperty.WIDTH: lambda: np.int32(image.shape[-1]),
-                    ImageProperty.HEIGHT: lambda: np.int32(image.shape[-2]),
-                    ImageProperty.SIZE: lambda: np.int32(image.shape[-1] * image.shape[-2]),
-                    ImageProperty.ASPECT_RATIO: lambda: image.shape[-1] / np.int32(image.shape[-2]),
-                    ImageProperty.CHANNELS: lambda: image.shape[-3],
-                    ImageProperty.DEPTH: lambda: get_bitdepth(image).depth,
-                }
-            )
-            self.results.append(results)
-
-
-class ImageVisualsMetric(BaseStatsMetric):
-    """
-    Calculates specified visual image properties
-
-    Parameters
-    ----------
-    flags: ImageVisuals
-        Property(ies) to calculate for each image
-    """
-
-    def __init__(self, flags: ImageVisuals = ImageVisuals.ALL):
-        super().__init__(flags)
-
-    def update(self, images: Iterable[ArrayLike]) -> None:
-        for image in to_numpy_iter(images):
-            results = self._map(
-                {
-                    ImageVisuals.BRIGHTNESS: lambda: np.mean(rescale(image)),
-                    ImageVisuals.BLURRINESS: lambda: np.std(edge_filter(np.mean(image, axis=0))),
-                    ImageVisuals.MISSING: lambda: np.sum(np.isnan(image)),
-                    ImageVisuals.ZERO: lambda: np.int32(np.count_nonzero(image == 0)),
-                }
-            )
-            self.results.append(results)
 
+QUARTILES = (0, 25, 50, 75, 100)
 
-class ImageStatisticsMetric(BaseStatsMetric):
+IMAGESTATS_FN_MAP: Dict[ImageStat, Callable[[NDArray], Any]] = {
+    ImageStat.XXHASH: lambda x: xxhash(x),
+    ImageStat.PCHASH: lambda x: pchash(x),
+    ImageStat.WIDTH: lambda x: np.uint16(x.shape[-1]),
+    ImageStat.HEIGHT: lambda x: np.uint16(x.shape[-2]),
+    ImageStat.CHANNELS: lambda x: np.uint8(x.shape[-3]),
+    ImageStat.SIZE: lambda x: np.uint32(np.prod(x.shape[-2:])),
+    ImageStat.ASPECT_RATIO: lambda x: np.float16(x.shape[-1] / x.shape[-2]),
+    ImageStat.DEPTH: lambda x: np.uint8(get_bitdepth(x).depth),
+    ImageStat.BRIGHTNESS: lambda x: np.float16(np.mean(x)),
+    ImageStat.BLURRINESS: lambda x: np.float16(np.std(edge_filter(np.mean(x, axis=0)))),
+    ImageStat.MISSING: lambda x: np.float16(np.sum(np.isnan(x)) / np.prod(x.shape[-2:])),
+    ImageStat.ZERO: lambda x: np.float16(np.count_nonzero(x == 0) / np.prod(x.shape[-2:])),
+    ImageStat.MEAN: lambda x: np.float16(np.mean(x)),
+    ImageStat.STD: lambda x: np.float16(np.std(x)),
+    ImageStat.VAR: lambda x: np.float16(np.var(x)),
+    ImageStat.SKEW: lambda x: np.float16(skew(x.ravel())),
+    ImageStat.KURTOSIS: lambda x: np.float16(kurtosis(x.ravel())),
+    ImageStat.PERCENTILES: lambda x: np.float16(np.percentile(x, q=QUARTILES)),
+    ImageStat.HISTOGRAM: lambda x: np.uint32(np.histogram(x, 256, (0, 1))[0]),
+    ImageStat.ENTROPY: lambda x: np.float16(entropy(x)),
+}
+
+CHANNELSTATS_FN_MAP: Dict[ImageStat, Callable[[NDArray], Any]] = {
+    ImageStat.MEAN: lambda x: np.float16(np.mean(x, axis=1)),
+    ImageStat.STD: lambda x: np.float16(np.std(x, axis=1)),
+    ImageStat.VAR: lambda x: np.float16(np.var(x, axis=1)),
+    ImageStat.SKEW: lambda x: np.float16(skew(x, axis=1)),
+    ImageStat.KURTOSIS: lambda x: np.float16(kurtosis(x, axis=1)),
+    ImageStat.PERCENTILES: lambda x: np.float16(np.percentile(x, q=QUARTILES, axis=1).T),
+    ImageStat.HISTOGRAM: lambda x: np.uint32(np.apply_along_axis(lambda y: np.histogram(y, 256, (0, 1))[0], 1, x)),
+    ImageStat.ENTROPY: lambda x: np.float16(entropy(x, axis=1)),
+}
+
+
+def run_stats(
+    images: Iterable[ArrayLike],
+    flags: ImageStat,
+    fn_map: Dict[ImageStat, Callable[[NDArray], Any]],
+    flatten: bool,
+):
+    verify_supported(flags, fn_map)
+    flag_dict = to_distinct(flags)
+
+    results_list: List[Dict[str, NDArray]] = []
+    for image in to_numpy_iter(images):
+        normalized = normalize_image_shape(image)
+        scaled = None
+        hist = None
+        output: Dict[str, NDArray] = {}
+        for flag, stat in flag_dict.items():
+            if flag & (ImageStat.ALL_PIXELSTATS | ImageStat.BRIGHTNESS):
+                if scaled is None:
+                    scaled = rescale(normalized).reshape(image.shape[0], -1) if flatten else rescale(normalized)
+                if flag & (ImageStat.HISTOGRAM | ImageStat.ENTROPY):
+                    if hist is None:
+                        hist = fn_map[ImageStat.HISTOGRAM](scaled)
+                    output[stat] = hist if flag & ImageStat.HISTOGRAM else fn_map[flag](hist)
+                else:
+                    output[stat] = fn_map[flag](scaled)
+            else:
+                output[stat] = fn_map[flag](normalized)
+        results_list.append(output)
+    return results_list
+
+
+@set_metadata("dataeval.metrics")
+def imagestats(images: Iterable[ArrayLike], flags: ImageStat = ImageStat.ALL_STATS) -> StatsOutput:
     """
-    Calculates descriptive statistics for each image
+    Calculates image and pixel statistics for each image
 
     Parameters
     ----------
-    flags: ImageStatistics
-        Statistic(s) to calculate for each image
+    images : Iterable[ArrayLike]
+        Images to run statistical tests on
+    flags : ImageStat, default ImageStat.ALL_STATS
+        Metric(s) to calculate for each image
+
+    Returns
+    -------
+    Dict[str, Any]
     """
-
-    def __init__(self, flags: ImageStatistics = ImageStatistics.ALL):
-        super().__init__(flags)
-
-    def update(self, images: Iterable[ArrayLike]) -> None:
-        for image in to_numpy_iter(images):
-            scaled = rescale(image)
-            if (ImageStatistics.HISTOGRAM | ImageStatistics.ENTROPY) & self.flags:
-                hist = np.histogram(scaled, bins=256, range=(0, 1))[0]
-
-            results = self._map(
-                {
-                    ImageStatistics.MEAN: lambda: np.mean(scaled),
-                    ImageStatistics.STD: lambda: np.std(scaled),
-                    ImageStatistics.VAR: lambda: np.var(scaled),
-                    ImageStatistics.SKEW: lambda: np.float32(skew(scaled.ravel())),
-                    ImageStatistics.KURTOSIS: lambda: np.float32(kurtosis(scaled.ravel())),
-                    ImageStatistics.PERCENTILES: lambda: np.percentile(scaled, q=QUARTILES),
-                    ImageStatistics.HISTOGRAM: lambda: hist,
-                    ImageStatistics.ENTROPY: lambda: np.float32(entropy(hist)),
-                }
-            )
-            self.results.append(results)
-
-
-class ChannelStatisticsMetric(BaseStatsMetric):
+    stats = run_stats(images, flags, IMAGESTATS_FN_MAP, False)
+    output = {}
+    length = len(stats)
+    for i, results in enumerate(stats):
+        for stat, result in results.items():
+            if not isinstance(result, (np.ndarray, np.generic)):
+                output.setdefault(stat, []).append(result)
+            else:
+                shape = () if np.isscalar(result) else result.shape
+                output.setdefault(stat, np.empty((length,) + shape))[i] = result
+    return StatsOutput(**populate_defaults(output, StatsOutput))
+
+
+@set_metadata("dataeval.metrics")
+def channelstats(images: Iterable[ArrayLike], flags=ImageStat.ALL_PIXELSTATS) -> StatsOutput:
     """
-    Calculates descriptive statistics for each image per channel
+    Calculates pixel statistics for each image per channel
 
     Parameters
     ----------
-    flags: ImageStatistics
+    images : Iterable[ArrayLike]
+        Images to run statistical tests on
+    flags: ImageStat, default ImageStat.ALL_PIXELSTATS
         Statistic(s) to calculate for each image per channel
-    """
-
-    def __init__(self, flags: ImageStatistics = ImageStatistics.ALL):
-        super().__init__(flags)
-
-    def update(self, images: Iterable[ArrayLike]) -> None:
-        for image in to_numpy_iter(images):
-            scaled = rescale(image)
-            flattened = scaled.reshape(image.shape[0], -1)
-
-            if (ImageStatistics.HISTOGRAM | ImageStatistics.ENTROPY) & self.flags:
-                hist = np.apply_along_axis(lambda x: np.histogram(x, bins=256, range=(0, 1))[0], 1, flattened)
+        Only flags in the ImageStat.ALL_PIXELSTATS category are supported
 
-            results = self._map(
-                {
-                    ImageStatistics.MEAN: lambda: np.mean(flattened, axis=1),
-                    ImageStatistics.STD: lambda: np.std(flattened, axis=1),
-                    ImageStatistics.VAR: lambda: np.var(flattened, axis=1),
-                    ImageStatistics.SKEW: lambda: skew(flattened, axis=1),
-                    ImageStatistics.KURTOSIS: lambda: kurtosis(flattened, axis=1),
-                    ImageStatistics.PERCENTILES: lambda: np.percentile(flattened, q=QUARTILES, axis=1).T,
-                    ImageStatistics.HISTOGRAM: lambda: hist,
-                    ImageStatistics.ENTROPY: lambda: entropy(hist, axis=1),
-                }
-            )
-            self.results.append(results)
-
-
-class BaseAggregateMetric(BaseStatsMetric, Generic[TFlag]):
-    FLAG_METRIC_MAP: Dict[type, type]
-    DEFAULT_FLAGS: Sequence[TFlag]
-
-    def __init__(self, flags: Optional[Union[TFlag, Sequence[TFlag]]] = None):
-        flag_dict = {}
-        for flag in flags if isinstance(flags, Sequence) else self.DEFAULT_FLAGS if not flags else [flags]:
-            flag_dict[type(flag)] = flag_dict.setdefault(type(flag), type(flag)(0)) | flag
-        self._metrics_dict = {
-            metric: []
-            for metric in (
-                self.FLAG_METRIC_MAP[flag_class](flag) for flag_class, flag in flag_dict.items() if flag.value != 0
-            )
-        }
-
-
-class ImageStats(BaseAggregateMetric):
-    """
-    Calculates various image property statistics
-
-    Parameters
-    ----------
-    flags: [ImageHash | ImageProperty | ImageStatistics | ImageVisuals], default None
-        Metric(s) to calculate for each image per channel - calculates all metrics if None
+    Returns
+    -------
+    Dict[str, Any]
     """
-
-    FLAG_METRIC_MAP = {
-        ImageHash: ImageHashMetric,
-        ImageProperty: ImagePropertyMetric,
-        ImageStatistics: ImageStatisticsMetric,
-        ImageVisuals: ImageVisualsMetric,
-    }
-    DEFAULT_FLAGS = [ImageHash.ALL, ImageProperty.ALL, ImageStatistics.ALL, ImageVisuals.ALL]
-
-    def __init__(self, flags: Optional[Union[ImageStatsFlags, Sequence[ImageStatsFlags]]] = None):
-        super().__init__(flags)
-        self._length = 0
-
-    def update(self, images: Iterable[ArrayLike]) -> None:
-        for image in to_numpy_iter(images):
-            self._length += 1
-            img = normalize_image_shape(image)
-            for metric in self._metrics_dict:
-                metric.update([img])
-
-    def compute(self) -> Dict[str, Any]:
-        for metric in self._metrics_dict:
-            self._metrics_dict[metric] = metric.results
-
-        stats = {}
-        for metric, results in self._metrics_dict.items():
-            for i, result in enumerate(results):
-                for stat in metric._keys():
-                    value = result[stat]
-                    if not isinstance(value, (np.ndarray, np.generic)):
-                        if stat not in stats:
-                            stats[stat] = []
-                        stats[stat].append(result[stat])
-                    else:
-                        if stat not in stats:
-                            shape = () if np.isscalar(result[stat]) else result[stat].shape
-                            stats[stat] = np.empty((self._length,) + shape)
-                        stats[stat][i] = result[stat]
-        return stats
-
-    def reset(self):
-        self._length = 0
-        for metric in self._metrics_dict:
-            metric.reset()
-            self._metrics_dict[metric] = []
-
-
-class ChannelStats(BaseAggregateMetric):
-    FLAG_METRIC_MAP = {ImageStatistics: ChannelStatisticsMetric}
-    DEFAULT_FLAGS = [ImageStatistics.ALL]
-    IDX_MAP = "idx_map"
-
-    def __init__(self, flags: Optional[ImageStatistics] = None) -> None:
-        super().__init__(flags)
-
-    def update(self, images: Iterable[ArrayLike]) -> None:
-        for image in to_numpy_iter(images):
-            img = normalize_image_shape(image)
-            for metric in self._metrics_dict:
-                metric.update([img])
-
-        for metric in self._metrics_dict:
-            self._metrics_dict[metric] = metric.results
-
-    def compute(self) -> Dict[str, Any]:
-        # Aggregate all metrics into a single dictionary
-        stats = {}
-        channel_stats = set()
-        for metric, results in self._metrics_dict.items():
-            for i, result in enumerate(results):
-                for stat in metric._keys():
-                    channel_stats.update(metric._keys())
-                    channels = result[stat].shape[0]
-                    stats.setdefault(self.IDX_MAP, {}).setdefault(channels, {})[i] = None
-                    stats.setdefault(stat, {}).setdefault(channels, []).append(result[stat])
-
-        # Concatenate list of channel statistics numpy
-        for stat in channel_stats:
-            for channel in stats[stat]:
-                stats[stat][channel] = np.array(stats[stat][channel]).T
-
-        for channel in stats[self.IDX_MAP]:
-            stats[self.IDX_MAP][channel] = list(stats[self.IDX_MAP][channel].keys())
-
-        return stats
-
-    def reset(self) -> None:
-        for metric in self._metrics_dict:
-            metric.reset()
-            self._metrics_dict[metric] = []
+    stats = run_stats(images, flags, CHANNELSTATS_FN_MAP, True)
+
+    output = {}
+    for i, results in enumerate(stats):
+        for stat, result in results.items():
+            channels = result.shape[0]
+            output.setdefault(stat, {}).setdefault(channels, []).append(result)
+            output.setdefault(CH_IDX_MAP, {}).setdefault(channels, {})[i] = None
+
+    # Concatenate list of channel statistics numpy
+    for stat in output:
+        if stat == CH_IDX_MAP:
+            continue
+        for channel in output[stat]:
+            output[stat][channel] = np.array(output[stat][channel]).T
+
+    for channel in output[CH_IDX_MAP]:
+        output[CH_IDX_MAP][channel] = list(output[CH_IDX_MAP][channel].keys())
+
+    return StatsOutput(**populate_defaults(output, StatsOutput))

dataeval/_internal/metrics/uap.py

@@ -4,15 +4,17 @@ FR Test Statistic based estimate for the upperbound
 average precision using empirical mean precision
 """
 
-from typing import NamedTuple
+from dataclasses import dataclass
 
 from numpy.typing import ArrayLike
 from sklearn.metrics import average_precision_score
 
 from dataeval._internal.interop import to_numpy
+from dataeval._internal.output import OutputMetadata, set_metadata
 
 
-class UAPOutput(NamedTuple):
+@dataclass(frozen=True)
+class UAPOutput(OutputMetadata):
     """
     Attributes
     ----------
@@ -23,6 +25,7 @@ class UAPOutput(NamedTuple):
     uap: float
 
 
+@set_metadata("dataeval.metrics")
 def uap(labels: ArrayLike, scores: ArrayLike) -> UAPOutput:
     """
     FR Test Statistic based estimate of the empirical mean precision for