dataeval 0.81.0__py3-none-any.whl → 0.82.1__py3-none-any.whl

dataeval/outputs/_linters.py

@@ -0,0 +1,186 @@
+from __future__ import annotations
+
+__all__ = []
+
+import contextlib
+from dataclasses import dataclass
+from typing import Generic, TypeVar, Union
+
+from typing_extensions import TypeAlias
+
+with contextlib.suppress(ImportError):
+    import pandas as pd
+
+from dataeval.outputs._base import Output
+from dataeval.outputs._stats import DimensionStatsOutput, LabelStatsOutput, PixelStatsOutput, VisualStatsOutput
+
+DuplicateGroup: TypeAlias = list[int]
+DatasetDuplicateGroupMap: TypeAlias = dict[int, DuplicateGroup]
+TIndexCollection = TypeVar("TIndexCollection", DuplicateGroup, DatasetDuplicateGroupMap)
+
+IndexIssueMap: TypeAlias = dict[int, dict[str, float]]
+OutlierStatsOutput: TypeAlias = Union[DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput]
+TIndexIssueMap = TypeVar("TIndexIssueMap", IndexIssueMap, list[IndexIssueMap])
+
+
+@dataclass(frozen=True)
+class DuplicatesOutput(Output, Generic[TIndexCollection]):
+    """
+    Output class for :class:`.Duplicates` lint detector.
+
+    Attributes
+    ----------
+    exact : list[list[int] | dict[int, list[int]]]
+        Indices of images that are exact matches
+    near: list[list[int] | dict[int, list[int]]]
+        Indices of images that are near matches
+
+    Notes
+    -----
+    - For a single dataset, indices are returned as a list of index groups.
+    - For multiple datasets, indices are returned as dictionaries where the key is the
+      index of the dataset, and the value is the list index groups from that dataset.
+    """
+
+    exact: list[TIndexCollection]
+    near: list[TIndexCollection]
+
+
+def _reorganize_by_class_and_metric(result: IndexIssueMap, lstats: LabelStatsOutput):
+    """Flip result from grouping by image to grouping by class and metric"""
+    metrics = {}
+    class_wise = {label: {} for label in lstats.class_names}
+
+    # Group metrics and calculate class-wise counts
+    for img, group in result.items():
+        for extreme in group:
+            metrics.setdefault(extreme, []).append(img)
+            for i, images in enumerate(lstats.image_indices_per_class):
+                if img in images:
+                    class_wise[lstats.class_names[i]][extreme] = class_wise[lstats.class_names[i]].get(extreme, 0) + 1
+
+    return metrics, class_wise
+
+
+def _create_table(metrics, class_wise):
+    """Create table for displaying the results"""
+    max_class_length = max(len(str(label)) for label in class_wise) + 2
+    max_total = max(len(metrics[group]) for group in metrics) + 2
+
+    table_header = " | ".join(
+        [f"{'Class':>{max_class_length}}"]
+        + [f"{group:^{max(5, len(str(group))) + 2}}" for group in sorted(metrics.keys())]
+        + [f"{'Total':<{max_total}}"]
+    )
+    table_rows = []
+
+    for class_cat, results in class_wise.items():
+        table_value = [f"{class_cat:>{max_class_length}}"]
+        total = 0
+        for group in sorted(metrics.keys()):
+            count = results.get(group, 0)
+            table_value.append(f"{count:^{max(5, len(str(group))) + 2}}")
+            total += count
+        table_value.append(f"{total:^{max_total}}")
+        table_rows.append(" | ".join(table_value))
+
+    table = [table_header] + table_rows
+    return table
+
+
+def _create_pandas_dataframe(class_wise):
+    """Create data for pandas dataframe"""
+    data = []
+    for label, metrics_dict in class_wise.items():
+        row = {"Class": label}
+        total = sum(metrics_dict.values())
+        row.update(metrics_dict)  # Add metric counts
+        row["Total"] = total
+        data.append(row)
+    return data
+
+
+@dataclass(frozen=True)
+class OutliersOutput(Output, Generic[TIndexIssueMap]):
+    """
+    Output class for :class:`.Outliers` lint detector.
+
+    Attributes
+    ----------
+    issues : dict[int, dict[str, float]] | list[dict[int, dict[str, float]]]
+        Indices of image Outliers with their associated issue type and calculated values.
+
+    - For a single dataset, a dictionary containing the indices of outliers and
+      a dictionary showing the issues and calculated values for the given index.
+    - For multiple stats outputs, a list of dictionaries containing the indices of
+      outliers and their associated issues and calculated values.
+    """
+
+    issues: TIndexIssueMap
+
+    def __len__(self) -> int:
+        if isinstance(self.issues, dict):
+            return len(self.issues)
+        else:
+            return sum(len(d) for d in self.issues)
+
+    def to_table(self, labelstats: LabelStatsOutput) -> str:
+        """
+        Formats the outlier output results as a table.
+
+        Parameters
+        ----------
+        labelstats : LabelStatsOutput
+            Output of :func:`.labelstats`
+
+        Returns
+        -------
+        str
+        """
+        if isinstance(self.issues, dict):
+            metrics, classwise = _reorganize_by_class_and_metric(self.issues, labelstats)
+            listed_table = _create_table(metrics, classwise)
+            table = "\n".join(listed_table)
+        else:
+            outertable = []
+            for d in self.issues:
+                metrics, classwise = _reorganize_by_class_and_metric(d, labelstats)
+                listed_table = _create_table(metrics, classwise)
+                str_table = "\n".join(listed_table)
+                outertable.append(str_table)
+            table = "\n\n".join(outertable)
+        return table
+
+    def to_dataframe(self, labelstats: LabelStatsOutput) -> pd.DataFrame:
+        """
+        Exports the outliers output results to a pandas DataFrame.
+
+        Parameters
+        ----------
+        labelstats : LabelStatsOutput
+            Output of :func:`.labelstats`
+
+        Returns
+        -------
+        pd.DataFrame
+
+        Notes
+        -----
+        This method requires `pandas <https://pandas.pydata.org/>`_ to be installed.
+        """
+        import pandas as pd
+
+        if isinstance(self.issues, dict):
+            _, classwise = _reorganize_by_class_and_metric(self.issues, labelstats)
+            data = _create_pandas_dataframe(classwise)
+            df = pd.DataFrame(data)
+        else:
+            df_list = []
+            for i, d in enumerate(self.issues):
+                _, classwise = _reorganize_by_class_and_metric(d, labelstats)
+                data = _create_pandas_dataframe(classwise)
+                single_df = pd.DataFrame(data)
+                single_df["Dataset"] = i
+                df_list.append(single_df)
+            df = pd.concat(df_list)
+        return df

dataeval/outputs/_metadata.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+__all__ = []
+
+from typing import NamedTuple
+
+from dataeval.outputs._base import MappingOutput, SequenceOutput
+
+
+class MostDeviatedFactorsOutput(SequenceOutput[tuple[str, float]]):
+    """
+    Output class for results of :func:`.most_deviated_factors` for OOD samples with metadata.
+
+    Attributes
+    ----------
+    value : tuple[str, float]
+        A tuple of the factor name and deviation of the highest metadata deviation
+    """
+
+
+class MetadataDistanceValues(NamedTuple):
+    """
+    Statistics comparing metadata distance.
+
+    Attributes
+    ----------
+    statistic : float
+        the KS statistic
+    location : float
+        The value at which the KS statistic has its maximum, measured in IQR-normalized units relative
+        to the median of the reference distribution.
+    dist : float
+        The Earth Mover's Distance normalized by the interquartile range (IQR) of the reference
+    pvalue : float
+        The p-value from the KS two-sample test
+    """
+
+    statistic: float
+    location: float
+    dist: float
+    pvalue: float
+
+
+class MetadataDistanceOutput(MappingOutput[str, MetadataDistanceValues]):
+    """
+    Output class for results of ks_2samp featurewise comparisons of new metadata to reference metadata.
+
+    Attributes
+    ----------
+    key : str
+        Metadata feature names
+    value : :class:`.MetadataDistanceValues`
+        Output per feature name containing the statistic, statistic location, distance, and pvalue.
+    """

dataeval/{detectors/ood/output.py → outputs/_ood.py}

@@ -8,27 +8,7 @@ from typing import Literal
 import numpy as np
 from numpy.typing import NDArray
 
-from dataeval._output import Output
-
-
-@dataclass(frozen=True)
-class OODOutput(Output):
-    """
-    Output class for predictions from out-of-distribution detectors.
-
-    Attributes
-    ----------
-    is_ood : NDArray
-        Array of images that are detected as :term:Out-of-Distribution (OOD)`
-    instance_score : NDArray
-        Instance score of the evaluated dataset
-    feature_score : NDArray | None
-        Feature score, if available, of the evaluated dataset
-    """
-
-    is_ood: NDArray[np.bool_]
-    instance_score: NDArray[np.float32]
-    feature_score: NDArray[np.float32] | None
+from dataeval.outputs._base import Output
 
 
 @dataclass(frozen=True)
@@ -49,7 +29,7 @@ class OODScoreOutput(Output):
 
     def get(self, ood_type: Literal["instance", "feature"]) -> NDArray[np.float32]:
         """
-        Returns either the instance or feature score
+        Returns either the instance or feature score.
 
         Parameters
         ----------
@@ -61,3 +41,23 @@ class OODScoreOutput(Output):
             Either the instance or feature score based on input selection
         """
         return self.instance_score if ood_type == "instance" or self.feature_score is None else self.feature_score
+
+
+@dataclass(frozen=True)
+class OODOutput(Output):
+    """
+    Output class for predictions from out-of-distribution detectors.
+
+    Attributes
+    ----------
+    is_ood : NDArray
+        Array of images that are detected as :term:Out-of-Distribution (OOD)`
+    instance_score : NDArray
+        Instance score of the evaluated dataset
+    feature_score : NDArray | None
+        Feature score, if available, of the evaluated dataset
+    """
+
+    is_ood: NDArray[np.bool_]
+    instance_score: NDArray[np.float32]
+    feature_score: NDArray[np.float32] | None

dataeval/outputs/_stats.py

@@ -0,0 +1,393 @@
+from __future__ import annotations
+
+__all__ = []
+
+import contextlib
+from dataclasses import dataclass
+from typing import Any, Iterable, Optional, Union
+
+import numpy as np
+from numpy.typing import NDArray
+from typing_extensions import TypeAlias
+
+with contextlib.suppress(ImportError):
+    import pandas as pd
+
+from dataeval.outputs._base import Output
+from dataeval.utils._plot import channel_histogram_plot, histogram_plot
+
+OptionalRange: TypeAlias = Optional[Union[int, Iterable[int]]]
+
+SOURCE_INDEX = "source_index"
+BOX_COUNT = "box_count"
+
+
+@dataclass(frozen=True)
+class SourceIndex:
+    """
+    The indices of the source image, box and channel.
+
+    Attributes
+    ----------
+    image: int
+        Index of the source image
+    box : int | None
+        Index of the box of the source image (if applicable)
+    channel : int | None
+        Index of the channel of the source image (if applicable)
+    """
+
+    image: int
+    box: int | None
+    channel: int | None
+
+
+def matches(index: int | None, opt_range: OptionalRange) -> bool:
+    if index is None or opt_range is None:
+        return True
+    return index in opt_range if isinstance(opt_range, Iterable) else index == opt_range
+
+
+@dataclass(frozen=True)
+class BaseStatsOutput(Output):
+    """
+    Attributes
+    ----------
+    source_index : List[SourceIndex]
+        Mapping from statistic to source image, box and channel index
+    box_count : NDArray[np.uint16]
+    """
+
+    source_index: list[SourceIndex]
+    box_count: NDArray[np.uint16]
+
+    def __post_init__(self) -> None:
+        length = len(self.source_index)
+        bad = {k: len(v) for k, v in self.data().items() if k not in [SOURCE_INDEX, BOX_COUNT] and len(v) != length}
+        if bad:
+            raise ValueError(f"All values must have the same length as source_index. Bad values: {str(bad)}.")
+
+    def get_channel_mask(
+        self,
+        channel_index: OptionalRange,
+        channel_count: OptionalRange = None,
+    ) -> list[bool]:
+        """
+        Boolean mask for results filtered to specified channel index and optionally the count
+        of the channels per image.
+
+        Parameters
+        ----------
+        channel_index : int | Iterable[int] | None
+            Index or indices of channel(s) to filter for
+        channel_count : int | Iterable[int] | None
+            Optional count(s) of channels to filter for
+        """
+        mask: list[bool] = []
+        cur_mask: list[bool] = []
+        cur_image = 0
+        cur_max_channel = 0
+        for source_index in list(self.source_index) + [None]:
+            if source_index is None or source_index.image > cur_image:
+                mask.extend(cur_mask if matches(cur_max_channel + 1, channel_count) else [False for _ in cur_mask])
+                if source_index is not None:
+                    cur_image = source_index.image
+                    cur_max_channel = 0
+                    cur_mask.clear()
+            if source_index is not None:
+                cur_mask.append(matches(source_index.channel, channel_index))
+                cur_max_channel = max(cur_max_channel, source_index.channel or 0)
+        return mask
+
+    def __len__(self) -> int:
+        return len(self.source_index)
+
+    def _get_channels(
+        self, channel_limit: int | None = None, channel_index: int | Iterable[int] | None = None
+    ) -> tuple[int, list[bool] | None]:
+        source_index = self.data()[SOURCE_INDEX]
+        raw_channels = int(max([si.channel or 0 for si in source_index])) + 1
+        if isinstance(channel_index, int):
+            max_channels = 1 if channel_index < raw_channels else raw_channels
+            ch_mask = self.get_channel_mask(channel_index)
+        elif isinstance(channel_index, Iterable) and all(isinstance(val, int) for val in list(channel_index)):
+            max_channels = len(list(channel_index))
+            ch_mask = self.get_channel_mask(channel_index)
+        elif isinstance(channel_limit, int):
+            max_channels = channel_limit
+            ch_mask = self.get_channel_mask(None, channel_limit)
+        else:
+            max_channels = raw_channels
+            ch_mask = None
+
+        if max_channels > raw_channels:
+            max_channels = raw_channels
+        if ch_mask is not None and not any(ch_mask):
+            ch_mask = None
+
+        return max_channels, ch_mask
+
+    def factors(self) -> dict[str, NDArray[Any]]:
+        return {
+            k: v
+            for k, v in self.data().items()
+            if k not in (SOURCE_INDEX, BOX_COUNT) and isinstance(v, np.ndarray) and v[v != 0].size > 0 and v.ndim == 1
+        }
+
+    def plot(
+        self, log: bool, channel_limit: int | None = None, channel_index: int | Iterable[int] | None = None
+    ) -> None:
+        max_channels, ch_mask = self._get_channels(channel_limit, channel_index)
+        if max_channels == 1:
+            histogram_plot(self.factors(), log)
+        else:
+            channel_histogram_plot(self.factors(), log, max_channels, ch_mask)
+
+
+@dataclass(frozen=True)
+class DimensionStatsOutput(BaseStatsOutput):
+    """
+    Output class for :func:`.dimensionstats` stats metric.
+
+    Attributes
+    ----------
+    left : NDArray[np.int32]
+        Offsets from the left edge of images in pixels
+    top : NDArray[np.int32]
+        Offsets from the top edge of images in pixels
+    width : NDArray[np.uint32]
+        Width of the images in pixels
+    height : NDArray[np.uint32]
+        Height of the images in pixels
+    channels : NDArray[np.uint8]
+        Channel count of the images in pixels
+    size : NDArray[np.uint32]
+        Size of the images in pixels
+    aspect_ratio : NDArray[np.float16]
+        :term:`ASspect Ratio<Aspect Ratio>` of the images (width/height)
+    depth : NDArray[np.uint8]
+        Color depth of the images in bits
+    center : NDArray[np.uint16]
+        Offset from center in [x,y] coordinates of the images in pixels
+    distance : NDArray[np.float16]
+        Distance in pixels from center
+    """
+
+    left: NDArray[np.int32]
+    top: NDArray[np.int32]
+    width: NDArray[np.uint32]
+    height: NDArray[np.uint32]
+    channels: NDArray[np.uint8]
+    size: NDArray[np.uint32]
+    aspect_ratio: NDArray[np.float16]
+    depth: NDArray[np.uint8]
+    center: NDArray[np.int16]
+    distance: NDArray[np.float16]
+
+
+@dataclass(frozen=True)
+class HashStatsOutput(BaseStatsOutput):
+    """
+    Output class for :func:`.hashstats` stats metric.
+
+    Attributes
+    ----------
+    xxhash : List[str]
+        xxHash hash of the images as a hex string
+    pchash : List[str]
+        :term:`Perception-based Hash` of the images as a hex string
+    """
+
+    xxhash: list[str]
+    pchash: list[str]
+
+
+@dataclass(frozen=True)
+class LabelStatsOutput(Output):
+    """
+    Output class for :func:`.labelstats` stats metric.
+
+    Attributes
+    ----------
+    label_counts_per_class : dict[int, int]
+        Dictionary whose keys are the different label classes and
+        values are total counts of each class
+    label_counts_per_image : list[int]
+        Number of labels per image
+    image_counts_per_class : dict[int, int]
+        Dictionary whose keys are the different label classes and
+        values are total counts of each image the class is present in
+    image_indices_per_class : dict[int, list]
+        Dictionary whose keys are the different label classes and
+        values are lists containing the images that have that label
+    image_count : int
+        Total number of images present
+    class_count : int
+        Total number of classes present
+    label_count : int
+        Total number of labels present
+    class_names : list[str]
+    """
+
+    label_counts_per_class: list[int]
+    label_counts_per_image: list[int]
+    image_counts_per_class: list[int]
+    image_indices_per_class: list[list[int]]
+    image_count: int
+    class_count: int
+    label_count: int
+    class_names: list[str]
+
+    def to_table(self) -> str:
+        """
+        Formats the label statistics output results as a table.
+
+        Returns
+        -------
+        str
+        """
+        max_char = max(len(name) if isinstance(name, str) else name // 10 + 1 for name in self.class_names)
+        max_char = max(max_char, 5)
+        max_label = max(list(self.label_counts_per_class))
+        max_img = max(list(self.image_counts_per_class))
+        max_num = int(np.ceil(np.log10(max(max_label, max_img))))
+        max_num = max(max_num, 11)
+
+        # Display basic counts
+        table_str = [f"Class Count: {self.class_count}"]
+        table_str += [f"Label Count: {self.label_count}"]
+        table_str += [f"Average # Labels per Image: {round(np.mean(self.label_counts_per_image), 2)}"]
+        table_str += ["--------------------------------------"]
+
+        # Display counts per class
+        table_str += [f"{'Label':>{max_char}}: Total Count - Image Count"]
+        for cls in range(len(self.class_names)):
+            table_str += [
+                f"{self.class_names[cls]:>{max_char}}: {self.label_counts_per_class[cls]:^{max_num}}"
+                + " - "
+                + f"{self.image_counts_per_class[cls]:^{max_num}}".rstrip()
+            ]
+
+        return "\n".join(table_str)
+
+    def to_dataframe(self) -> pd.DataFrame:
+        """
+        Exports the label statistics output results to a pandas DataFrame.
+
+        Notes
+        -----
+        This method requires `pandas <https://pandas.pydata.org/>`_ to be installed.
+
+        Returns
+        -------
+        pd.DataFrame
+        """
+        import pandas as pd
+
+        total_count = []
+        image_count = []
+        for cls in range(len(self.class_names)):
+            total_count.append(self.label_counts_per_class[cls])
+            image_count.append(self.image_counts_per_class[cls])
+
+        return pd.DataFrame(
+            {
+                "Label": self.class_names,
+                "Total Count": total_count,
+                "Image Count": image_count,
+            }
+        )
+
+
+@dataclass(frozen=True)
+class PixelStatsOutput(BaseStatsOutput):
+    """
+    Output class for :func:`.pixelstats` stats metric.
+
+    Attributes
+    ----------
+    mean : NDArray[np.float16]
+        Mean of the pixel values of the images
+    std : NDArray[np.float16]
+        Standard deviation of the pixel values of the images
+    var : NDArray[np.float16]
+        :term:`Variance` of the pixel values of the images
+    skew : NDArray[np.float16]
+        Skew of the pixel values of the images
+    kurtosis : NDArray[np.float16]
+        Kurtosis of the pixel values of the images
+    histogram : NDArray[np.uint32]
+        Histogram of the pixel values of the images across 256 bins scaled between 0 and 1
+    entropy : NDArray[np.float16]
+        Entropy of the pixel values of the images
+    """
+
+    mean: NDArray[np.float16]
+    std: NDArray[np.float16]
+    var: NDArray[np.float16]
+    skew: NDArray[np.float16]
+    kurtosis: NDArray[np.float16]
+    histogram: NDArray[np.uint32]
+    entropy: NDArray[np.float16]
+
+
+@dataclass(frozen=True)
+class VisualStatsOutput(BaseStatsOutput):
+    """
+    Output class for :func:`.visualstats` stats metric.
+
+    Attributes
+    ----------
+    brightness : NDArray[np.float16]
+        Brightness of the images
+    contrast : NDArray[np.float16]
+        Image contrast ratio
+    darkness : NDArray[np.float16]
+        Darkness of the images
+    missing : NDArray[np.float16]
+        Percentage of the images with missing pixels
+    sharpness : NDArray[np.float16]
+        Sharpness of the images
+    zeros : NDArray[np.float16]
+        Percentage of the images with zero value pixels
+    percentiles : NDArray[np.float16]
+        Percentiles of the pixel values of the images with quartiles of (0, 25, 50, 75, 100)
+    """
+
+    brightness: NDArray[np.float16]
+    contrast: NDArray[np.float16]
+    darkness: NDArray[np.float16]
+    missing: NDArray[np.float16]
+    sharpness: NDArray[np.float16]
+    zeros: NDArray[np.float16]
+    percentiles: NDArray[np.float16]
+
+
+@dataclass(frozen=True)
+class ImageStatsOutput(DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput):
+    """
+    Output class for :func:`.imagestats` stats metric with `per_channel=False`.
+
+    This class represents the combined outputs of various stats functions against a
+    single dataset, such that each index across all stat outputs are representative
+    of the same source image. Modifying or mixing outputs will result in inaccurate
+    outlier calculations if not created correctly.
+
+    The attributes and methods are a superset of :class:`.DimensionStatsOutput`,
+    :class:`.PixelStatsOutput` and :class:`.VisualStatsOutput`.
+    """
+
+
+@dataclass(frozen=True)
+class ChannelStatsOutput(PixelStatsOutput, VisualStatsOutput):
+    """
+    Output class for :func:`.imagestats` stats metric with `per_channel=True`.
+
+    This class represents the outputs of various per-channel stats functions against
+    a single dataset, such that each index across all stat outputs are representative
+    of the same source image. Modifying or mixing outputs will result in inaccurate
+    outlier calculations if not created correctly.
+
+    The attributes and methods are a superset of :class:`.PixelStatsOutput` and
+    :class:`.VisualStatsOutput`.
+    """