dataeval 0.67.0__tar.gz → 0.68.0__tar.gz

{dataeval-0.67.0 → dataeval-0.68.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataeval
-Version: 0.67.0
+Version: 0.68.0
 Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
 Home-page: https://dataeval.ai/
 License: MIT

{dataeval-0.67.0 → dataeval-0.68.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dataeval"
-version = "0.67.0" # dynamic
+version = "0.68.0" # dynamic
 description = "DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks"
 license = "MIT"
 readme = "README.md"
@@ -174,7 +174,7 @@ docstring-code-format = true
 docstring-code-line-length = "dynamic"
 
 [tool.codespell]
-skip = './*env*,./prototype,./docs/.jupyter_cache,./.tox,CHANGELOG.md,poetry.lock,./output,*.html'
+skip = './*env*,./prototype,./output,./docs/_build,./docs/.jupyter_cache,CHANGELOG.md,poetry.lock,*.html'
 
 [build-system]
 requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"]

{dataeval-0.67.0 → dataeval-0.68.0}/src/dataeval/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.67.0"
+__version__ = "0.68.0"
 
 from importlib.util import find_spec
 

dataeval-0.68.0/src/dataeval/_internal/detectors/duplicates.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Generic, Iterable, Sequence, TypeVar, cast
+
+from numpy.typing import ArrayLike
+
+from dataeval._internal.detectors.merged_stats import combine_stats, get_dataset_step_from_idx
+from dataeval._internal.flags import ImageStat
+from dataeval._internal.metrics.stats import StatsOutput, imagestats
+from dataeval._internal.output import OutputMetadata, set_metadata
+
+DuplicateGroup = list[int]
+DatasetDuplicateGroupMap = dict[int, DuplicateGroup]
+TIndexCollection = TypeVar("TIndexCollection", DuplicateGroup, DatasetDuplicateGroupMap)
+
+
+@dataclass(frozen=True)
+class DuplicatesOutput(Generic[TIndexCollection], OutputMetadata):
+    """
+    Attributes
+    ----------
+    exact : list[list[int] | dict[int, list[int]]]
+        Indices of images that are exact matches
+    near: list[list[int] | dict[int, list[int]]]
+        Indices of images that are near matches
+
+    - For a single dataset, indices are returned as a list of index groups.
+    - For multiple datasets, indices are returned as dictionaries where the key is the
+      index of the dataset, and the value is the list index groups from that dataset.
+    """
+
+    exact: list[TIndexCollection]
+    near: list[TIndexCollection]
+
+
+class Duplicates:
+    """
+    Finds the duplicate images in a dataset using xxhash for exact duplicates
+    and pchash for near duplicates
+
+    Attributes
+    ----------
+    stats : StatsOutput
+        Output class of stats
+
+    Parameters
+    ----------
+    only_exact : bool, default False
+        Only inspect the dataset for exact image matches
+
+    Example
+    -------
+    Initialize the Duplicates class:
+
+    >>> dups = Duplicates()
+    """
+
+    def __init__(self, only_exact: bool = False):
+        self.stats: StatsOutput
+        self.only_exact = only_exact
+
+    def _get_duplicates(self) -> dict[str, list[list[int]]]:
+        stats_dict = self.stats.dict()
+        if "xxhash" in stats_dict:
+            exact_dict: dict[int, list] = {}
+            for i, value in enumerate(stats_dict["xxhash"]):
+                exact_dict.setdefault(value, []).append(i)
+            exact = [sorted(v) for v in exact_dict.values() if len(v) > 1]
+        else:
+            exact = []
+
+        if "pchash" in stats_dict and not self.only_exact:
+            near_dict: dict[int, list] = {}
+            for i, value in enumerate(stats_dict["pchash"]):
+                near_dict.setdefault(value, []).append(i)
+            near = [sorted(v) for v in near_dict.values() if len(v) > 1 and not any(set(v).issubset(x) for x in exact)]
+        else:
+            near = []
+
+        return {
+            "exact": sorted(exact),
+            "near": sorted(near),
+        }
+
+    @set_metadata("dataeval.detectors", ["only_exact"])
+    def evaluate(self, data: Iterable[ArrayLike] | StatsOutput | Sequence[StatsOutput]) -> DuplicatesOutput:
+        """
+        Returns duplicate image indices for both exact matches and near matches
+
+        Parameters
+        ----------
+        data : Iterable[ArrayLike], shape - (N, C, H, W) | StatsOutput | Sequence[StatsOutput]
+            A dataset of images in an ArrayLike format or the output(s) from an imagestats metric analysis
+
+        Returns
+        -------
+        DuplicatesOutput
+            List of groups of indices that are exact and near matches
+
+        See Also
+        --------
+        imagestats
+
+        Example
+        -------
+        >>> dups.evaluate(images)
+        DuplicatesOutput(exact=[[3, 20], [16, 37]], near=[[3, 20, 22], [12, 18], [13, 36], [14, 31], [17, 27], [19, 38, 47]])
+        """  # noqa: E501
+
+        stats, dataset_steps = combine_stats(data)
+
+        if isinstance(stats, StatsOutput):
+            if not stats.xxhash:
+                raise ValueError("StatsOutput must include xxhash information of the images.")
+            if not self.only_exact and not stats.pchash:
+                raise ValueError("StatsOutput must include pchash information of the images for near matches.")
+            self.stats = stats
+        else:
+            flags = ImageStat.XXHASH | (ImageStat(0) if self.only_exact else ImageStat.PCHASH)
+            self.stats = imagestats(cast(Iterable[ArrayLike], data), flags)
+
+        duplicates = self._get_duplicates()
+
+        # split up results from combined dataset into individual dataset buckets
+        if dataset_steps:
+            dup_list: list[list[int]]
+            for dup_type, dup_list in duplicates.items():
+                dup_list_dict = []
+                for idxs in dup_list:
+                    dup_dict = {}
+                    for idx in idxs:
+                        k, v = get_dataset_step_from_idx(idx, dataset_steps)
+                        dup_dict.setdefault(k, []).append(v)
+                    dup_list_dict.append(dup_dict)
+                duplicates[dup_type] = dup_list_dict
+
+        return DuplicatesOutput(**duplicates)

dataeval-0.68.0/src/dataeval/_internal/detectors/merged_stats.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from typing import Sequence, cast
+from warnings import warn
+
+import numpy as np
+
+from dataeval._internal.metrics.stats import StatsOutput
+from dataeval._internal.output import populate_defaults
+
+
+def add_stats(a: StatsOutput, b: StatsOutput) -> StatsOutput:
+    if not isinstance(a, StatsOutput) or not isinstance(b, StatsOutput):
+        raise TypeError(f"Cannot add object of type {type(a)} and type {type(b)}.")
+
+    a_dict = a.dict()
+    b_dict = b.dict()
+    a_keys = set(a_dict)
+    b_keys = set(b_dict)
+
+    missing_keys = a_keys - b_keys
+    if missing_keys:
+        raise ValueError(f"Required keys are missing: {missing_keys}.")
+
+    extra_keys = b_keys - a_keys
+    if extra_keys:
+        warn(f"Extraneous keys will be dropped: {extra_keys}.")
+
+    # perform add of multi-channel stats
+    if "ch_idx_map" in a_dict:
+        for k, v in a_dict.items():
+            if k == "ch_idx_map":
+                offset = sum([len(idxs) for idxs in v.values()])
+                for ch_k, ch_v in b_dict[k].items():
+                    if ch_k not in v:
+                        v[ch_k] = []
+                    a_dict[k][ch_k].extend([idx + offset for idx in ch_v])
+            else:
+                for ch_k in b_dict[k]:
+                    if ch_k not in v:
+                        v[ch_k] = b_dict[k][ch_k]
+                    else:
+                        v[ch_k] = np.concatenate((v[ch_k], b_dict[k][ch_k]), axis=1)
+    else:
+        for k in a_dict:
+            if isinstance(a_dict[k], list):
+                a_dict[k].extend(b_dict[k])
+            else:
+                a_dict[k] = np.concatenate((a_dict[k], b_dict[k]))
+
+    return StatsOutput(**populate_defaults(a_dict, StatsOutput))
+
+
+def combine_stats(stats) -> tuple[StatsOutput | None, list[int]]:
+    dataset_steps = []
+
+    if isinstance(stats, StatsOutput):
+        return stats, dataset_steps
+
+    output = None
+    if isinstance(stats, Sequence) and isinstance(stats[0], StatsOutput):
+        stats = cast(Sequence[StatsOutput], stats)
+        cur_len = 0
+        for s in stats:
+            output = s if output is None else add_stats(output, s)
+            cur_len += len(s)
+            dataset_steps.append(cur_len)
+
+    return output, dataset_steps
+
+
+def get_dataset_step_from_idx(idx: int, dataset_steps: list[int]) -> tuple[int, int]:
+    last_step = 0
+    for i, step in enumerate(dataset_steps):
+        if idx < step:
+            return i, idx - last_step
+        last_step = step
+    return -1, idx

{dataeval-0.67.0 → dataeval-0.68.0}/src/dataeval/_internal/detectors/outliers.py

@@ -1,27 +1,39 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import Iterable, Literal
+from typing import Iterable, Literal, Sequence, cast
+from warnings import warn
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
 
+from dataeval._internal.detectors.merged_stats import combine_stats, get_dataset_step_from_idx
 from dataeval._internal.flags import ImageStat, to_distinct, verify_supported
 from dataeval._internal.metrics.stats import StatsOutput, imagestats
 from dataeval._internal.output import OutputMetadata, set_metadata
 
+IndexIssueMap = dict[int, dict[str, float]]
+DatasetIndexIssueMap = dict[int, IndexIssueMap]
+"""
+Mapping of image indices to a dictionary of issue types and calculated values
+"""
+
 
 @dataclass(frozen=True)
 class OutliersOutput(OutputMetadata):
     """
     Attributes
     ----------
-    issues : Dict[int, Dict[str, float]]
-        Dictionary containing the indices of outliers and a dictionary showing
-        the issues and calculated values for the given index.
+    issues : dict[int, dict[str, float]] | dict[int, dict[int, dict[str, float]]]
+        Indices of image outliers with their associated issue type and calculated values.
+
+    - For a single dataset, a dictionary containing the indices of outliers and
+      a dictionary showing the issues and calculated values for the given index.
+    - For multiple datasets, a map of dataset indices to the indices of outliers
+      and their associated issues and calculated values.
     """
 
-    issues: dict[int, dict[str, float]]
+    issues: IndexIssueMap | DatasetIndexIssueMap
 
 
 def _get_outlier_mask(
@@ -64,7 +76,7 @@ class Outliers:
 
     Attributes
     ----------
-    stats : Dict[str, Any]
+    stats : dict[str, Any]
         Dictionary to hold the value of each metric for each image
 
     See Also
@@ -135,14 +147,14 @@ class Outliers:
         return dict(sorted(flagged_images.items()))
 
     @set_metadata("dataeval.detectors", ["flags", "outlier_method", "outlier_threshold"])
-    def evaluate(self, data: Iterable[ArrayLike] | StatsOutput) -> OutliersOutput:
+    def evaluate(self, data: Iterable[ArrayLike] | StatsOutput | Sequence[StatsOutput]) -> OutliersOutput:
         """
         Returns indices of outliers with the issues identified for each
 
         Parameters
         ----------
-        data : Iterable[ArrayLike], shape - (C, H, W) | StatsOutput
-            A dataset of images in an ArrayLike format or the output from an imagestats metric analysis
+        data : Iterable[ArrayLike], shape - (C, H, W) | StatsOutput | Sequence[StatsOutput]
+            A dataset of images in an ArrayLike format or the output(s) from an imagestats metric analysis
 
         Returns
         -------
@@ -157,13 +169,29 @@ class Outliers:
         >>> outliers.evaluate(images)
         OutliersOutput(issues={18: {'brightness': 0.78}, 25: {'brightness': 0.98}})
         """
-        if isinstance(data, StatsOutput):
-            flags = set(to_distinct(self.flags).values())
-            stats = set(data.dict())
-            missing = flags - stats
+        stats, dataset_steps = combine_stats(data)
+
+        if isinstance(stats, StatsOutput):
+            selected_flags = set(to_distinct(self.flags).values())
+            provided = set(stats.dict())
+            missing = selected_flags - provided
             if missing:
-                raise ValueError(f"StatsOutput is missing {missing} from the required stats: {flags}.")
-            self.stats = data
+                warn(
+                    f"StatsOutput provided {provided} and is missing {missing} \
+                        from the selected stat flags: {selected_flags}."
+                )
+            self.stats = stats
         else:
-            self.stats = imagestats(data, self.flags)
-        return OutliersOutput(self._get_outliers())
+            self.stats = imagestats(cast(Iterable[ArrayLike], data), self.flags)
+
+        outliers = self._get_outliers()
+
+        # split up results from combined dataset into individual dataset buckets
+        if dataset_steps:
+            out_dict = {}
+            for idx, issue in outliers.items():
+                k, v = get_dataset_step_from_idx(idx, dataset_steps)
+                out_dict.setdefault(k, {})[v] = issue
+            outliers = out_dict
+
+        return OutliersOutput(outliers)

{dataeval-0.67.0 → dataeval-0.68.0}/src/dataeval/_internal/metrics/balance.py

@@ -17,11 +17,17 @@ class BalanceOutput(OutputMetadata):
     """
     Attributes
     ----------
-    mutual_information : NDArray[np.float64]
+    balance : NDArray[np.float64]
         Estimate of mutual information between metadata factors and class label
+    factors : NDArray[np.float64]
+        Estimate of inter/intra-factor mutual information
+    classwise : NDArray[np.float64]
+        Estimate of mutual information between metadata factors and individual class labels
     """
 
-    mutual_information: NDArray[np.float64]
+    balance: NDArray[np.float64]
+    factors: NDArray[np.float64]
+    classwise: NDArray[np.float64]
 
 
 def validate_num_neighbors(num_neighbors: int) -> int:
@@ -77,17 +83,22 @@ def balance(class_labels: Sequence[int], metadata: list[dict], num_neighbors: in
     -------
     Return balance (mutual information) of factors with class_labels
 
-    >>> balance(class_labels, metadata).mutual_information[0]
-    array([0.99999822, 0.13363788, 0.        , 0.02994455])
+    >>> bal = balance(class_labels, metadata)
+    >>> bal.balance
+    array([0.99999822, 0.13363788, 0.04505382, 0.02994455])
 
-    Return balance (mutual information) of metadata factors with class_labels
-    and each other
+    Return intra/interfactor balance (mutual information)
 
-    >>> balance(class_labels, metadata).mutual_information
-    array([[0.99999822, 0.13363788, 0.        , 0.02994455],
-           [0.13363788, 0.99999843, 0.01389763, 0.09725766],
-           [0.        , 0.01389763, 0.48549233, 0.15314612],
-           [0.02994455, 0.09725766, 0.15314612, 0.99999856]])
+    >>> bal.factors
+    array([[0.99999843, 0.03510422, 0.09725766],
+           [0.03510422, 0.08433558, 0.15621459],
+           [0.09725766, 0.15621459, 0.99999856]])
+
+    Return classwise balance (mutual information) of factors with individual class_labels
+
+    >>> bal.classwise
+    array([[0.99999822, 0.13363788, 0.        , 0.        ],
+           [0.99999822, 0.13363788, 0.        , 0.        ]])
 
     See Also
     --------
@@ -102,13 +113,9 @@ def balance(class_labels: Sequence[int], metadata: list[dict], num_neighbors: in
     mi[:] = np.nan
 
     for idx in range(num_factors):
-        tgt = data[:, idx]
+        tgt = data[:, idx].astype(int)
 
         if is_categorical[idx]:
-            if tgt.dtype == float:
-                # map to unique integers if categorical
-                _, tgt = np.unique(tgt, return_inverse=True)
-            # categorical target
             mi[idx, :] = mutual_info_classif(
                 data,
                 tgt,
@@ -129,89 +136,40 @@ def balance(class_labels: Sequence[int], metadata: list[dict], num_neighbors: in
     norm_factor = 0.5 * np.add.outer(ent_all, ent_all) + 1e-6
     # in principle MI should be symmetric, but it is not in practice.
     nmi = 0.5 * (mi + mi.T) / norm_factor
+    balance = nmi[0]
+    factors = nmi[1:, 1:]
 
-    return BalanceOutput(nmi)
-
-
-@set_metadata("dataeval.metrics")
-def balance_classwise(class_labels: Sequence[int], metadata: list[dict], num_neighbors: int = 5) -> BalanceOutput:
-    """
-    Compute mutual information (analogous to correlation) between metadata factors
-    (class label, metadata, label/image properties) with individual class labels.
-
-    Parameters
-    ----------
-    class_labels: Sequence[int]
-        List of class labels for each image
-    metadata: List[Dict]
-        List of metadata factors for each image
-    num_neighbors: int, default 5
-        Number of nearest neighbors to use for computing MI between discrete
-        and continuous variables.
-
-    Notes
-    -----
-    We use `mutual_info_classif` from sklearn since class label is categorical.
-    `mutual_info_classif` outputs are consistent up to O(1e-4) and depend on a random
-    seed. MI is computed differently for categorical and continuous variables, so we
-    have to specify with is_categorical.
-
-    Returns
-    -------
-    BalanceOutput
-        (num_classes x num_factors) estimate of mutual information between
-        num_factors metadata factors and individual class labels.
-
-    Example
-    -------
-    Return classwise balance (mutual information) of factors with individual class_labels
-
-    >>> balance_classwise(class_labels, metadata).mutual_information
-    array([[0.13363788, 0.54085156, 0.        ],
-           [0.13363788, 0.54085156, 0.        ]])
-
-
-    See Also
-    --------
-    sklearn.feature_selection.mutual_info_classif
-    sklearn.feature_selection.mutual_info_regression
-    sklearn.metrics.mutual_info_score
-    compute_mutual_information
-    """
-    num_neighbors = validate_num_neighbors(num_neighbors)
-    data, names, is_categorical = preprocess_metadata(class_labels, metadata)
-    num_factors = len(names)
     # unique class labels
     class_idx = names.index("class_label")
-    class_data = data[:, class_idx]
+    class_data = data[:, class_idx].astype(int)
     u_cls = np.unique(class_data)
     num_classes = len(u_cls)
 
-    data_no_class = np.concatenate((data[:, :class_idx], data[:, (class_idx + 1) :]), axis=1)
-
     # assume class is a factor
-    mi = np.empty((num_classes, num_factors - 1))
-    mi[:] = np.nan
+    classwise_mi = np.empty((num_classes, num_factors))
+    classwise_mi[:] = np.nan
 
     # categorical variables, excluding class label
     cat_mask = np.concatenate((is_categorical[:class_idx], is_categorical[(class_idx + 1) :]), axis=0).astype(int)
 
+    tgt_bin = np.stack([class_data == cls for cls in u_cls]).T.astype(int)
+    ent_tgt_bin = entropy(
+        tgt_bin, names=[str(idx) for idx in range(num_classes)], is_categorical=[True for idx in range(num_classes)]
+    )
+
     # classification MI for discrete/categorical features
-    for idx, cls in enumerate(u_cls):
-        tgt = class_data == cls
+    for idx in range(num_classes):
+        # tgt = class_data == cls
         # units: nat
-        mi[idx, :] = mutual_info_classif(
-            data_no_class,
-            tgt,
+        classwise_mi[idx, :] = mutual_info_classif(
+            data,
+            tgt_bin[:, idx],
             discrete_features=cat_mask,  # type: ignore
             n_neighbors=num_neighbors,
             random_state=0,
         )
 
-    # let this recompute for all features including class label
-    ent_all = entropy(data, names, is_categorical)
-    ent_tgt = ent_all[class_idx]
-    ent_all = np.concatenate((ent_all[:class_idx], ent_all[(class_idx + 1) :]), axis=0)
-    norm_factor = 0.5 * np.add.outer(ent_tgt, ent_all) + 1e-6
-    nmi = mi / norm_factor
-    return BalanceOutput(nmi)
+    norm_factor = 0.5 * np.add.outer(ent_tgt_bin, ent_all) + 1e-6
+    classwise = classwise_mi / norm_factor
+
+    return BalanceOutput(balance, factors, classwise)

{dataeval-0.67.0 → dataeval-0.68.0}/src/dataeval/_internal/metrics/coverage.py

@@ -66,27 +66,22 @@ def coverage(
 
     Note
     ----
-    Embeddings should be on the unit interval.
+    Embeddings should be on the unit interval [0-1].
 
     Example
     -------
-    >>> coverage(embeddings)
-    CoverageOutput(indices=array([], dtype=int64), radii=array([0.59307666, 0.56956307, 0.56328616, 0.70660265, 0.57778087,
-           0.53738624, 0.58968217, 1.27721334, 0.84378694, 0.67767021,
-           0.69680335, 1.35532621, 0.59764166, 0.8691945 , 0.83627602,
-           0.84187303, 0.62212358, 1.09039732, 0.67956797, 0.60134383,
-           0.83713908, 0.91784263, 1.12901193, 0.73907618, 0.63943983,
-           0.61188447, 0.47872713, 0.57207771, 0.92885883, 0.54750511,
-           0.83015726, 1.20721778, 0.50421928, 0.98312246, 0.59764166,
-           0.61009202, 0.73864073, 1.0381061 , 0.77598609, 0.72984036,
-           0.67573006, 0.48056064, 1.00050879, 0.89532971, 0.58395529,
-           0.95954793, 0.60134383, 1.10096454, 0.51955314, 0.73038702]), critical_value=0)
+    >>> results = coverage(embeddings)
+    >>> results.indices
+    array([447, 412,   8,  32,  63])
+    >>> results.critical_value
+    0.8459038956941765
 
     Reference
     ---------
     This implementation is based on https://dl.acm.org/doi/abs/10.1145/3448016.3457315.
+
     [1] Seymour Sudman. 1976. Applied sampling. Academic Press New York (1976).
-    """  # noqa: E501
+    """
 
     # Calculate distance matrix, look at the (k+1)th farthest neighbor for each image.
     embeddings = to_numpy(embeddings)
@@ -105,8 +100,9 @@ def coverage(
         pvals = np.where(crit > rho)[0]
     elif radius_type == "adaptive":
         # Use data adaptive cutoff as rho
-        rho = int(n * percent)
-        pvals = np.argsort(crit)[::-1][:rho]
+        selection = int(max(n * percent, 1))
+        pvals = np.argsort(crit)[::-1][:selection]
+        rho = float(np.mean(np.sort(crit)[::-1][selection - 1 : selection + 1]))
     else:
         raise ValueError(f"{radius_type} is an invalid radius type. Expected 'adaptive' or 'naive'")
     return CoverageOutput(pvals, crit, rho)

{dataeval-0.67.0 → dataeval-0.68.0}/src/dataeval/_internal/metrics/diversity.py

@@ -17,9 +17,12 @@ class DiversityOutput(OutputMetadata):
     ----------
     diversity_index : NDArray[np.float64]
         Diversity index for classes and factors
+    classwise : NDArray[np.float64]
+        Classwise diversity index [n_class x n_factor]
     """
 
     diversity_index: NDArray[np.float64]
+    classwise: NDArray[np.float64]
 
 
 def diversity_shannon(
@@ -139,9 +142,11 @@ def diversity(
     class_labels: Sequence[int], metadata: list[dict], method: Literal["shannon", "simpson"] = "simpson"
 ) -> DiversityOutput:
     """
-    Compute diversity for discrete/categorical variables and, through standard
+    Compute diversity and classwise diversity for discrete/categorical variables and, through standard
     histogram binning, for continuous variables.
 
+    We define diversity as a normalized form of the inverse Simpson diversity index.
+
     diversity = 1 implies that samples are evenly distributed across a particular factor
     diversity = 0 implies that all samples belong to one category/bin
 
@@ -157,89 +162,45 @@ def diversity(
     Notes
     -----
     - For continuous variables, histogram bins are chosen automatically. See numpy.histogram for details.
+    - The expression is undefined for q=1, but it approaches the Shannon entropy in the limit.
+    - If there is only one category, the diversity index takes a value of 1 = 1/N = 1/1. Entropy will take a value of 0.
 
     Returns
     -------
     DiversityOutput
-        Diversity index per column of self.data or each factor in self.names
+        Diversity index per column of self.data or each factor in self.names and
+        classwise diversity [n_class x n_factor]
 
     Example
     -------
     Compute Simpson diversity index of metadata and class labels
 
-    >>> diversity(class_labels, metadata, method="simpson").diversity_index
+    >>> div_simp = diversity(class_labels, metadata, method="simpson")
+    >>> div_simp.diversity_index
     array([0.18103448, 0.18103448, 0.88636364])
 
-    Compute Shannon diversity index of metadata and class labels
-
-    >>> diversity(class_labels, metadata, method="shannon").diversity_index
-    array([0.37955133, 0.37955133, 0.96748876])
-
-
-    See Also
-    --------
-    numpy.histogram
-    """
-    diversity_fn = get_method(DIVERSITY_FN_MAP, method)
-    data, names, is_categorical = preprocess_metadata(class_labels, metadata)
-    diversity_index = diversity_fn(data, names, is_categorical, None).astype(np.float64)
-    return DiversityOutput(diversity_index)
-
-
-@set_metadata("dataeval.metrics")
-def diversity_classwise(
-    class_labels: Sequence[int], metadata: list[dict], method: Literal["shannon", "simpson"] = "simpson"
-) -> DiversityOutput:
-    """
-    Compute diversity for discrete/categorical variables and, through standard
-    histogram binning, for continuous variables.
-
-    We define diversity as a normalized form of the inverse Simpson diversity
-    index.
-
-    diversity = 1 implies that samples are evenly distributed across a particular factor
-    diversity = 0 implies that all samples belong to one category/bin
-
-    Parameters
-    ----------
-    class_labels: Sequence[int]
-        List of class labels for each image
-    metadata: List[Dict]
-        List of metadata factors for each image
-    method: Literal["shannon", "simpson"], default "simpson"
-        Indicates which diversity index should be computed
-
-    Notes
-    -----
-    - For continuous variables, histogram bins are chosen automatically. See numpy.histogram for details.
-    - If there is only one category, the diversity index takes a value of 0.
-
-    Returns
-    -------
-    DiversityOutput
-        Diversity index [n_class x n_factor]
-
-    Example
-    -------
-    Compute classwise Simpson diversity index of metadata and class labels
-
-    >>> diversity_classwise(class_labels, metadata, method="simpson").diversity_index
+    >>> div_simp.classwise
     array([[0.17241379, 0.39473684],
            [0.2       , 0.2       ]])
 
-    Compute classwise Shannon diversity index of metadata and class labels
+    Compute Shannon diversity index of metadata and class labels
 
-    >>> diversity_classwise(class_labels, metadata, method="shannon").diversity_index
+    >>> div_shan = diversity(class_labels, metadata, method="shannon")
+    >>> div_shan.diversity_index
+    array([0.37955133, 0.37955133, 0.96748876])
+
+    >>> div_shan.classwise
     array([[0.43156028, 0.83224889],
            [0.57938016, 0.57938016]])
 
-
     See Also
     --------
     numpy.histogram
     """
     diversity_fn = get_method(DIVERSITY_FN_MAP, method)
     data, names, is_categorical = preprocess_metadata(class_labels, metadata)
+    diversity_index = diversity_fn(data, names, is_categorical, None).astype(np.float64)
+
     class_idx = names.index("class_label")
     class_lbl = data[:, class_idx]
 
@@ -251,4 +212,5 @@ def diversity_classwise(
         subset_mask = class_lbl == cls
         diversity[idx, :] = diversity_fn(data, names, is_categorical, subset_mask)
     div_no_class = np.concatenate((diversity[:, :class_idx], diversity[:, (class_idx + 1) :]), axis=1)
-    return DiversityOutput(div_no_class)
+
+    return DiversityOutput(diversity_index, div_no_class)

{dataeval-0.67.0 → dataeval-0.68.0}/src/dataeval/_internal/metrics/stats.py

@@ -89,6 +89,16 @@ class StatsOutput(OutputMetadata):
     def dict(self):
         return {k: v for k, v in self.__dict__.items() if not k.startswith("_") and len(v) > 0}
 
+    def __len__(self) -> int:
+        if self.ch_idx_map:
+            return sum([len(idxs) for idxs in self.ch_idx_map.values()])
+        else:
+            for a in self.__annotations__:
+                attr = getattr(self, a, None)
+                if attr is not None and hasattr(a, "__len__") and len(attr) > 0:
+                    return len(attr)
+        return 0
+
 
 QUARTILES = (0, 25, 50, 75, 100)
 

{dataeval-0.67.0 → dataeval-0.68.0}/src/dataeval/_internal/output.py

@@ -11,7 +11,7 @@ from dataeval import __version__
 
 class OutputMetadata:
     _name: str
-    _execution_time: str
+    _execution_time: datetime
     _execution_duration: float
     _arguments: dict[str, str]
     _state: dict[str, str]

{dataeval-0.67.0 → dataeval-0.68.0}/src/dataeval/metrics/bias/__init__.py

@@ -1,14 +1,12 @@
-from dataeval._internal.metrics.balance import balance, balance_classwise
+from dataeval._internal.metrics.balance import balance
 from dataeval._internal.metrics.coverage import coverage
-from dataeval._internal.metrics.diversity import diversity, diversity_classwise
+from dataeval._internal.metrics.diversity import diversity
 from dataeval._internal.metrics.parity import label_parity, parity
 
 __all__ = [
     "balance",
-    "balance_classwise",
     "coverage",
     "diversity",
-    "diversity_classwise",
     "label_parity",
     "parity",
 ]

dataeval-0.67.0/src/dataeval/_internal/detectors/duplicates.py

@@ -1,109 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from typing import Iterable
-
-from numpy.typing import ArrayLike
-
-from dataeval._internal.flags import ImageStat
-from dataeval._internal.metrics.stats import StatsOutput, imagestats
-from dataeval._internal.output import OutputMetadata, set_metadata
-
-
-@dataclass(frozen=True)
-class DuplicatesOutput(OutputMetadata):
-    """
-    Attributes
-    ----------
-    exact : List[List[int]]
-        Indices of images that are exact matches
-    near: List[List[int]]
-        Indices of images that are near matches
-    """
-
-    exact: list[list[int]]
-    near: list[list[int]]
-
-
-class Duplicates:
-    """
-    Finds the duplicate images in a dataset using xxhash for exact duplicates
-    and pchash for near duplicates
-
-    Attributes
-    ----------
-    stats : StatsOutput
-        Output class of stats
-
-    Parameters
-    ----------
-    only_exact : bool, default False
-        Only inspect the dataset for exact image matches
-
-    Example
-    -------
-    Initialize the Duplicates class:
-
-    >>> dups = Duplicates()
-    """
-
-    def __init__(self, only_exact: bool = False):
-        self.stats: StatsOutput
-        self.only_exact = only_exact
-
-    def _get_duplicates(self) -> dict[str, list[list[int]]]:
-        stats_dict = self.stats.dict()
-        if "xxhash" in stats_dict:
-            exact = {}
-            for i, value in enumerate(stats_dict["xxhash"]):
-                exact.setdefault(value, []).append(i)
-            exact = [v for v in exact.values() if len(v) > 1]
-        else:
-            exact = []
-
-        if "pchash" in stats_dict and not self.only_exact:
-            near = {}
-            for i, value in enumerate(stats_dict["pchash"]):
-                near.setdefault(value, []).append(i)
-            near = [v for v in near.values() if len(v) > 1 and not any(set(v).issubset(x) for x in exact)]
-        else:
-            near = []
-
-        return {
-            "exact": sorted(exact),
-            "near": sorted(near),
-        }
-
-    @set_metadata("dataeval.detectors", ["only_exact"])
-    def evaluate(self, data: Iterable[ArrayLike] | StatsOutput) -> DuplicatesOutput:
-        """
-        Returns duplicate image indices for both exact matches and near matches
-
-        Parameters
-        ----------
-        data : Iterable[ArrayLike], shape - (N, C, H, W) | StatsOutput
-            A dataset of images in an ArrayLike format or the output from an imagestats metric analysis
-
-        Returns
-        -------
-        DuplicatesOutput
-            List of groups of indices that are exact and near matches
-
-        See Also
-        --------
-        imagestats
-
-        Example
-        -------
-        >>> dups.evaluate(images)
-        DuplicatesOutput(exact=[[3, 20], [16, 37]], near=[[3, 20, 22], [12, 18], [13, 36], [14, 31], [17, 27], [19, 38, 47]])
-        """  # noqa: E501
-        if isinstance(data, StatsOutput):
-            if not data.xxhash:
-                raise ValueError("StatsOutput must include xxhash information of the images.")
-            if not self.only_exact and not data.pchash:
-                raise ValueError("StatsOutput must include pchash information of the images for near matches.")
-            self.stats = data
-        else:
-            self.stats = imagestats(data, ImageStat.XXHASH | (ImageStat(0) if self.only_exact else ImageStat.PCHASH))
-        return DuplicatesOutput(**self._get_duplicates())