dataeval 0.86.0__py3-none-any.whl → 0.86.2__py3-none-any.whl

dataeval/detectors/linters/outliers.py

@@ -13,31 +13,31 @@ from dataeval.metrics.stats._imagestats import imagestats
 from dataeval.outputs import DimensionStatsOutput, ImageStatsOutput, OutliersOutput, PixelStatsOutput, VisualStatsOutput
 from dataeval.outputs._base import set_metadata
 from dataeval.outputs._linters import IndexIssueMap, OutlierStatsOutput
-from dataeval.outputs._stats import BOX_COUNT, SOURCE_INDEX
+from dataeval.outputs._stats import BASE_ATTRS
 from dataeval.typing import ArrayLike, Dataset
 
 
 def _get_outlier_mask(
     values: NDArray, method: Literal["zscore", "modzscore", "iqr"], threshold: float | None
 ) -> NDArray:
+    values = values.astype(np.float64)
     if method == "zscore":
         threshold = threshold if threshold else 3.0
         std = np.std(values)
         abs_diff = np.abs(values - np.mean(values))
         return std != 0 and (abs_diff / std) > threshold
-    elif method == "modzscore":
+    if method == "modzscore":
         threshold = threshold if threshold else 3.5
         abs_diff = np.abs(values - np.median(values))
         med_abs_diff = np.median(abs_diff) if np.median(abs_diff) != 0 else np.mean(abs_diff)
         mod_z_score = 0.6745 * abs_diff / med_abs_diff
         return mod_z_score > threshold
-    elif method == "iqr":
+    if method == "iqr":
         threshold = threshold if threshold else 1.5
         qrt = np.percentile(values, q=(25, 75), method="midpoint")
         iqr = (qrt[1] - qrt[0]) * threshold
         return (values < (qrt[0] - iqr)) | (values > (qrt[1] + iqr))
-    else:
-        raise ValueError("Outlier method must be 'zscore' 'modzscore' or 'iqr'.")
+    raise ValueError("Outlier method must be 'zscore' 'modzscore' or 'iqr'.")
 
 
 class Outliers:
@@ -103,7 +103,7 @@ class Outliers:
         use_visual: bool = True,
         outlier_method: Literal["zscore", "modzscore", "iqr"] = "modzscore",
         outlier_threshold: float | None = None,
-    ):
+    ) -> None:
         self.stats: ImageStatsOutput
         self.use_dimension = use_dimension
         self.use_pixel = use_pixel
@@ -114,7 +114,7 @@ class Outliers:
     def _get_outliers(self, stats: dict) -> dict[int, dict[str, float]]:
         flagged_images: dict[int, dict[str, float]] = {}
         for stat, values in stats.items():
-            if stat in (SOURCE_INDEX, BOX_COUNT):
+            if stat in BASE_ATTRS:
                 continue
             if values.ndim == 1:
                 mask = _get_outlier_mask(values.astype(np.float64), self.outlier_method, self.outlier_threshold)

dataeval/metadata/_distance.py

@@ -80,14 +80,17 @@ def metadata_distance(metadata1: Metadata, metadata2: Metadata) -> MetadataDista
     MetadataDistanceValues(statistic=1.0, location=0.44354838709677413, dist=2.7, pvalue=0.0)
     """
 
-    _compare_keys(metadata1.continuous_factor_names, metadata2.continuous_factor_names)
-    fnames = metadata1.continuous_factor_names
+    _compare_keys(metadata1.factor_names, metadata2.factor_names)
+    cont_fnames = metadata1.get_factors_by_type("continuous")
 
-    cont1 = np.atleast_2d(metadata1.continuous_data)  # (S, F)
-    cont2 = np.atleast_2d(metadata2.continuous_data)  # (S, F)
+    if not cont_fnames:
+        return MetadataDistanceOutput({})
 
-    _validate_factors_and_data(fnames, cont1)
-    _validate_factors_and_data(fnames, cont2)
+    cont1 = np.atleast_2d(metadata1.dataframe[cont_fnames].to_numpy())  # (S, F)
+    cont2 = np.atleast_2d(metadata2.dataframe[cont_fnames].to_numpy())  # (S, F)
+
+    _validate_factors_and_data(cont_fnames, cont1)
+    _validate_factors_and_data(cont_fnames, cont2)
 
     N = len(cont1)
     M = len(cont2)
@@ -104,7 +107,7 @@ def metadata_distance(metadata1: Metadata, metadata2: Metadata) -> MetadataDista
     results: dict[str, MetadataDistanceValues] = {}
 
     # Per factor
-    for i, fname in enumerate(fnames):
+    for i, fname in enumerate(cont_fnames):
         fdata1 = cont1[:, i]  # (S, 1)
         fdata2 = cont2[:, i]  # (S, 1)
 

dataeval/metadata/_ood.py

@@ -15,95 +15,6 @@ from dataeval.outputs import MostDeviatedFactorsOutput, OODOutput, OODPredictorO
 from dataeval.outputs._base import set_metadata
 
 
-def _combine_discrete_continuous(metadata: Metadata) -> tuple[list[str], NDArray[np.float64]]:
-    """Combines the discrete and continuous data of a :class:`Metadata` object
-
-    Returns
-    -------
-    Tuple[list[str], NDArray]
-        The combined list of factors names and the combined discrete and continuous data
-
-    Note
-    ----
-    Discrete and continuous data must have the same number of samples
-    """
-    names = []
-    data = []
-
-    if metadata.discrete_factor_names and metadata.discrete_data.size != 0:
-        names.extend(metadata.discrete_factor_names)
-        data.append(metadata.discrete_data)
-
-    if metadata.continuous_factor_names and metadata.continuous_data.size != 0:
-        names.extend(metadata.continuous_factor_names)
-        data.append(metadata.continuous_data)
-
-    return names, np.hstack(data, dtype=np.float64) if data else np.array([], dtype=np.float64)
-
-
-def _combine_metadata(
-    metadata_1: Metadata, metadata_2: Metadata
-) -> tuple[list[str], list[NDArray[np.float64 | np.int64]], list[NDArray[np.int64 | np.float64]]]:
-    """
-    Combines the factor names and data arrays of metadata_1 and metadata_2 when the names
-    match exactly and data has the same number of columns (factors).
-
-    Parameters
-    ----------
-    metadata_1 : Metadata
-        The set of factor names used as reference to determine the correct factor names and length of data
-    metadata_2 : Metadata
-        The compared set of factor names and data that must match metadata_1
-
-    Returns
-    -------
-    list[str]
-        The combined discrete and continuous factor names in that order.
-    list[NDArray]
-        Combined discrete and continuous data of metadata_1
-    list[NDArray]
-        Combined discrete and continuous data of metadata_2
-
-    Raises
-    ------
-    ValueError
-        If keys do not match in metadata_1 and metadata_2
-    ValueError
-        If the length of keys do not match the length of the data
-    """
-    factor_names: list[str] = []
-    m1_data: list[NDArray[np.int64 | np.float64]] = []
-    m2_data: list[NDArray[np.int64 | np.float64]] = []
-
-    # Both metadata must have the same number of factors (cols), but not necessarily samples (row)
-    if metadata_1.total_num_factors != metadata_2.total_num_factors:
-        raise ValueError(
-            f"Number of factors differs between metadata_1 ({metadata_1.total_num_factors}) "
-            f"and metadata_2 ({metadata_2.total_num_factors})"
-        )
-
-    # Validate and attach discrete data
-    if metadata_1.discrete_factor_names:
-        _compare_keys(metadata_1.discrete_factor_names, metadata_2.discrete_factor_names)
-        _validate_factors_and_data(metadata_1.discrete_factor_names, metadata_1.discrete_data)
-
-        factor_names.extend(metadata_1.discrete_factor_names)
-        m1_data.append(metadata_1.discrete_data)
-        m2_data.append(metadata_2.discrete_data)
-
-    # Validate and attach continuous data
-    if metadata_1.continuous_factor_names:
-        _compare_keys(metadata_1.continuous_factor_names, metadata_2.continuous_factor_names)
-        _validate_factors_and_data(metadata_1.continuous_factor_names, metadata_1.continuous_data)
-
-        factor_names.extend(metadata_1.continuous_factor_names)
-        m1_data.append(metadata_1.continuous_data)
-        m2_data.append(metadata_2.continuous_data)
-
-    # Turns list of discrete and continuous into one array
-    return factor_names, m1_data, m2_data
-
-
 def _calc_median_deviations(reference: NDArray, test: NDArray) -> NDArray:
     """
     Calculates deviations of the test data from the median of the reference data
@@ -207,16 +118,13 @@ def find_most_deviated_factors(
     if not any(ood_mask):
         return MostDeviatedFactorsOutput([])
 
-    # Combines reference and test factor names and data if exists and match exactly
-    # shape -> (samples, factors)
-    factor_names, md_1, md_2 = _combine_metadata(
-        metadata_1=metadata_ref,
-        metadata_2=metadata_tst,
-    )
+    factor_names = metadata_ref.factor_names
+    ref_data = metadata_ref.factor_data
+    tst_data = metadata_tst.factor_data
 
-    # Stack discrete and continuous factors as separate factors. Must have equal sample counts
-    ref_data = np.hstack(md_1) if md_1 else np.array([])  # (S, Fd + Fc)
-    tst_data = np.hstack(md_2) if md_2 else np.array([])  # (S, Fd + Fc)
+    _compare_keys(factor_names, metadata_tst.factor_names)
+    _validate_factors_and_data(factor_names, ref_data)
+    _validate_factors_and_data(factor_names, tst_data)
 
     if len(ref_data) < 3:
         warnings.warn(
@@ -256,6 +164,7 @@ which is what many library functions return, multiply it by _NATS2BITS to get it
 """
 
 
+@set_metadata
 def find_ood_predictors(
     metadata: Metadata,
     ood: OODOutput,
@@ -305,8 +214,8 @@ def find_ood_predictors(
 
     ood_mask: NDArray[np.bool_] = ood.is_ood
 
-    discrete_features_count = len(metadata.discrete_factor_names)
-    factors, data = _combine_discrete_continuous(metadata)  # (F, ), (S, F) => F = Fd + Fc
+    factors = metadata.factor_names
+    data = metadata.factor_data
 
     # No metadata correlated with out of distribution data, return 0.0 for all factors
     if not any(ood_mask):
@@ -320,14 +229,13 @@ def find_ood_predictors(
     # Calculate mean, std of each factor over all samples
     scaled_data = (data - np.mean(data, axis=0)) / np.std(data, axis=0, ddof=1)  # (S, F)
 
-    discrete_features = np.zeros_like(factors, dtype=np.bool_)
-    discrete_features[:discrete_features_count] = True
+    discrete_features = [info.factor_type != "continuous" for info in metadata.factor_info.values()]
 
     mutual_info_values = (
         mutual_info_classif(
             X=scaled_data,
             y=ood_mask,
-            discrete_features=discrete_features,  # type: ignore -> sklearn issue - NDArray[bool] not of accepted type Union[ArrayLike, 'auto']
+            discrete_features=discrete_features,  # type: ignore - sklearn function not typed
             random_state=get_seed(),
         )
         * _NATS2BITS

dataeval/metrics/bias/_balance.py

@@ -68,22 +68,20 @@ def balance(
 
     >>> bal = balance(metadata)
     >>> bal.balance
-    array([1.   , 0.249, 0.03 , 0.134, 0.   , 0.   ])
+    array([1.   , 0.134, 0.   , 0.   ])
 
     Return intra/interfactor balance (mutual information)
 
     >>> bal.factors
-    array([[1.   , 0.314, 0.269, 0.852, 0.367],
-           [0.314, 1.   , 0.097, 0.158, 1.98 ],
-           [0.269, 0.097, 1.   , 0.037, 0.015],
-           [0.852, 0.158, 0.037, 0.475, 0.255],
-           [0.367, 1.98 , 0.015, 0.255, 1.063]])
+    array([[1.   , 0.017, 0.015],
+           [0.017, 0.445, 0.245],
+           [0.015, 0.245, 1.063]])
 
     Return classwise balance (mutual information) of factors with individual class_labels
 
     >>> bal.classwise
-    array([[1.   , 0.249, 0.03 , 0.134, 0.   , 0.   ],
-           [1.   , 0.249, 0.03 , 0.134, 0.   , 0.   ]])
+    array([[1.   , 0.134, 0.   , 0.   ],
+           [1.   , 0.134, 0.   , 0.   ]])
 
 
     See Also
@@ -92,41 +90,39 @@ def balance(
     sklearn.feature_selection.mutual_info_regression
     sklearn.metrics.mutual_info_score
     """
-    if not metadata.discrete_factor_names and not metadata.continuous_factor_names:
+    if not metadata.factor_names:
         raise ValueError("No factors found in provided metadata.")
 
     num_neighbors = _validate_num_neighbors(num_neighbors)
 
-    num_factors = metadata.total_num_factors
-    is_discrete = [True] * (len(metadata.discrete_factor_names) + 1) + [False] * len(metadata.continuous_factor_names)
+    data = metadata.discretized_data
+    factor_types = {"class_label": "categorical"} | {k: v.factor_type for k, v in metadata.factor_info.items()}
+    is_discrete = [factor_type != "continuous" for factor_type in factor_types.values()]
+    num_factors = len(factor_types)
+
     mi = np.full((num_factors, num_factors), np.nan, dtype=np.float32)
-    data = np.hstack((metadata.class_labels[:, np.newaxis], metadata.discrete_data))
-    discretized_data = data
-    if len(metadata.continuous_data):
-        data = np.hstack((data, metadata.continuous_data))
-        discrete_idx = [metadata.discrete_factor_names.index(name) for name in metadata.continuous_factor_names]
-        discretized_data = np.hstack((discretized_data, metadata.discrete_data[:, discrete_idx]))
-
-    for idx in range(num_factors):
-        if idx >= len(metadata.discrete_factor_names) + 1:
-            mi[idx, :] = mutual_info_regression(
+    data = np.hstack((metadata.class_labels[:, np.newaxis], data))
+
+    for idx, factor_type in enumerate(factor_types.values()):
+        if factor_type != "continuous":
+            mi[idx, :] = mutual_info_classif(
                 data,
                 data[:, idx],
-                discrete_features=is_discrete,  # type: ignore
+                discrete_features=is_discrete,  # type: ignore - sklearn function not typed
                 n_neighbors=num_neighbors,
                 random_state=get_seed(),
             )
         else:
-            mi[idx, :] = mutual_info_classif(
+            mi[idx, :] = mutual_info_regression(
                 data,
                 data[:, idx],
-                discrete_features=is_discrete,  # type: ignore
+                discrete_features=is_discrete,  # type: ignore - sklearn function not typed
                 n_neighbors=num_neighbors,
                 random_state=get_seed(),
             )
 
     # Normalization via entropy
-    bin_cnts = get_counts(discretized_data)
+    bin_cnts = get_counts(data)
     ent_factor = sp.stats.entropy(bin_cnts, axis=0)
     norm_factor = 0.5 * np.add.outer(ent_factor, ent_factor) + EPSILON
 
@@ -149,7 +145,7 @@ def balance(
         classwise_mi[idx, :] = mutual_info_classif(
             data,
             tgt_bin[:, idx],
-            discrete_features=is_discrete,  # type: ignore
+            discrete_features=is_discrete,  # type: ignore - sklearn function not typed
             n_neighbors=num_neighbors,
             random_state=get_seed(),
         )
@@ -161,12 +157,6 @@ def balance(
     classwise = classwise_mi / norm_factor
 
     # Grabbing factor names for plotting function
-    factor_names = ["class"]
-    for name in metadata.discrete_factor_names:
-        if name in metadata.continuous_factor_names:
-            name = name + "-discrete"
-        factor_names.append(name)
-    for name in metadata.continuous_factor_names:
-        factor_names.append(name + "-continuous")
+    factor_names = ["class_label"] + metadata.factor_names
 
     return BalanceOutput(balance, factors, classwise, factor_names, metadata.class_names)

dataeval/metrics/bias/_diversity.py

@@ -138,43 +138,45 @@ def diversity(
 
     >>> div_simp = diversity(metadata, method="simpson")
     >>> div_simp.diversity_index
-    array([0.6  , 0.809, 1.   , 0.8  ])
+    array([0.6  , 0.8  , 0.809, 1.   ])
 
     >>> div_simp.classwise
-    array([[0.5  , 0.8  , 0.8  ],
-           [0.63 , 0.976, 0.528]])
+    array([[0.8  , 0.5  , 0.8  ],
+           [0.528, 0.63 , 0.976]])
 
     Compute Shannon diversity index of metadata and class labels
 
     >>> div_shan = diversity(metadata, method="shannon")
     >>> div_shan.diversity_index
-    array([0.811, 0.943, 1.   , 0.918])
+    array([0.811, 0.918, 0.943, 1.   ])
 
     >>> div_shan.classwise
-    array([[0.683, 0.918, 0.918],
-           [0.814, 0.991, 0.764]])
+    array([[0.918, 0.683, 0.918],
+           [0.764, 0.814, 0.991]])
 
     See Also
     --------
     scipy.stats.entropy
     """
-    if not metadata.discrete_factor_names and not metadata.continuous_factor_names:
+    if not metadata.factor_names:
         raise ValueError("No factors found in provided metadata.")
 
     diversity_fn = get_method(_DIVERSITY_FN_MAP, method)
-    discretized_data = np.hstack((metadata.class_labels[:, np.newaxis], metadata.discrete_data))
-    cnts = get_counts(discretized_data)
+    discretized_data = metadata.discretized_data
+    factor_names = metadata.factor_names
+    class_lbl = metadata.class_labels
+
+    class_labels_with_discretized_data = np.hstack((class_lbl[:, np.newaxis], discretized_data))
+    cnts = get_counts(class_labels_with_discretized_data)
     num_bins = np.bincount(np.nonzero(cnts)[1])
     diversity_index = diversity_fn(cnts, num_bins)
 
-    class_lbl = metadata.class_labels
-
     u_classes = np.unique(class_lbl)
-    num_factors = len(metadata.discrete_factor_names)
+    num_factors = len(factor_names)
     classwise_div = np.full((len(u_classes), num_factors), np.nan)
     for idx, cls in enumerate(u_classes):
         subset_mask = class_lbl == cls
-        cls_cnts = get_counts(metadata.discrete_data[subset_mask], min_num_bins=cnts.shape[0])
+        cls_cnts = get_counts(discretized_data[subset_mask], min_num_bins=cnts.shape[0])
         classwise_div[idx, :] = diversity_fn(cls_cnts, num_bins[1:])
 
-    return DiversityOutput(diversity_index, classwise_div, metadata.discrete_factor_names, metadata.class_names)
+    return DiversityOutput(diversity_index, classwise_div, factor_names, metadata.class_names)

dataeval/metrics/bias/_parity.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 __all__ = []
 
 import warnings
+from collections import defaultdict
 from typing import Any
 
 import numpy as np
@@ -241,13 +242,13 @@ def parity(metadata: Metadata) -> ParityOutput:
     >>> parity(metadata)
     ParityOutput(score=array([7.357, 5.467, 0.515]), p_value=array([0.289, 0.243, 0.773]), factor_names=['age', 'income', 'gender'], insufficient_data={'age': {3: {'artist': 4}, 4: {'artist': 4, 'teacher': 3}}, 'income': {1: {'artist': 3}}})
     """  # noqa: E501
-    if not metadata.discrete_factor_names and not metadata.continuous_factor_names:
+    if not metadata.factor_names:
         raise ValueError("No factors found in provided metadata.")
 
-    chi_scores = np.zeros(metadata.discrete_data.shape[1])
+    chi_scores = np.zeros(metadata.discretized_data.shape[1])
     p_values = np.zeros_like(chi_scores)
-    insufficient_data = {}
-    for i, col_data in enumerate(metadata.discrete_data.T):
+    insufficient_data: defaultdict[str, defaultdict[int, dict[str, int]]] = defaultdict(lambda: defaultdict(dict))
+    for i, col_data in enumerate(metadata.discretized_data.T):
         # Builds a contingency matrix where entry at index (r,c) represents
         # the frequency of current_factor_name achieving value unique_factor_values[r]
         # at a data point with class c.
@@ -257,30 +258,29 @@ def parity(metadata: Metadata) -> ParityOutput:
         # Determines if any frequencies are too low
         counts = np.nonzero(contingency_matrix < 5)
         unique_factor_values = np.unique(col_data)
-        current_factor_name = metadata.discrete_factor_names[i]
+        current_factor_name = metadata.factor_names[i]
         for int_factor, int_class in zip(counts[0], counts[1]):
             if contingency_matrix[int_factor, int_class] > 0:
                 factor_category = unique_factor_values[int_factor].item()
-                if current_factor_name not in insufficient_data:
-                    insufficient_data[current_factor_name] = {}
-                if factor_category not in insufficient_data[current_factor_name]:
-                    insufficient_data[current_factor_name][factor_category] = {}
                 class_name = metadata.class_names[int_class]
                 class_count = contingency_matrix[int_factor, int_class].item()
                 insufficient_data[current_factor_name][factor_category][class_name] = class_count
 
         # This deletes rows containing only zeros,
         # because scipy.stats.chi2_contingency fails when there are rows containing only zeros.
-        rowsums = np.sum(contingency_matrix, axis=1)
-        rowmask = np.nonzero(rowsums)[0]
-        contingency_matrix = contingency_matrix[rowmask]
+        contingency_matrix = contingency_matrix[np.any(contingency_matrix, axis=1)]
 
-        chi2, p, _, _ = chi2_contingency(contingency_matrix)
-
-        chi_scores[i] = chi2
-        p_values[i] = p
+        chi_scores[i], p_values[i] = chi2_contingency(contingency_matrix)[:2]
 
     if insufficient_data:
-        warnings.warn("Some factors did not meet the recommended 5 occurrences for each value-label combination.")
+        warnings.warn(
+            f"Factors {list(insufficient_data)} did not meet the recommended "
+            "5 occurrences for each value-label combination."
+        )
 
-    return ParityOutput(chi_scores, p_values, metadata.discrete_factor_names, insufficient_data)
+    return ParityOutput(
+        score=chi_scores,
+        p_value=p_values,
+        factor_names=metadata.factor_names,
+        insufficient_data={k: dict(v) for k, v in insufficient_data.items()},
+    )

dataeval/metrics/estimators/_divergence.py

@@ -38,8 +38,7 @@ def divergence_mst(data: NDArray[np.float64], labels: NDArray[np.int_]) -> int:
     """
     mst = minimum_spanning_tree(data).toarray()
     edgelist = np.transpose(np.nonzero(mst))
-    errors = np.sum(labels[edgelist[:, 0]] != labels[edgelist[:, 1]])
-    return errors
+    return np.sum(labels[edgelist[:, 0]] != labels[edgelist[:, 1]])
 
 
 def divergence_fnn(data: NDArray[np.float64], labels: NDArray[np.int_]) -> int:
@@ -59,8 +58,7 @@ def divergence_fnn(data: NDArray[np.float64], labels: NDArray[np.int_]) -> int:
         Number of label errors when finding nearest neighbors
     """
     nn_indices = compute_neighbors(data, data)
-    errors = np.sum(np.abs(labels[nn_indices] - labels))
-    return errors
+    return np.sum(np.abs(labels[nn_indices] - labels))
 
 
 _DIVERGENCE_FN_MAP = {"FNN": divergence_fnn, "MST": divergence_mst}