dataeval 0.85.0__py3-none-any.whl → 0.86.1__py3-none-any.whl

dataeval/detectors/drift/_nml/_result.py

@@ -0,0 +1,97 @@
+"""
+Contains the results of the data reconstruction drift calculation and provides plotting functionality.
+
+Source code derived from NannyML 0.13.0
+https://github.com/NannyML/nannyml/blob/main/nannyml/base.py
+
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from __future__ import annotations
+
+import copy
+from abc import ABC, abstractmethod
+from typing import NamedTuple, Sequence
+
+import pandas as pd
+from typing_extensions import Self
+
+from dataeval.outputs._base import GenericOutput
+
+
+class Metric(NamedTuple):
+    display_name: str
+    column_name: str
+
+
+class AbstractResult(GenericOutput[pd.DataFrame]):
+    def __init__(self, results_data: pd.DataFrame) -> None:
+        self._data = results_data.copy(deep=True)
+
+    def data(self) -> pd.DataFrame:
+        return self.to_df()
+
+    @property
+    def empty(self) -> bool:
+        return self._data is None or self._data.empty
+
+    def __len__(self) -> int:
+        return 0 if self.empty else len(self._data)
+
+    def to_df(self, multilevel: bool = True) -> pd.DataFrame:
+        """Export results to pandas dataframe."""
+        if multilevel:
+            return self._data
+        column_names = [
+            "_".join(col).replace("chunk_chunk_chunk", "chunk").replace("chunk_chunk", "chunk")
+            for col in self._data.columns.values
+        ]
+        single_level_data = self._data.copy(deep=True)
+        single_level_data.columns = column_names
+        return single_level_data
+
+    def filter(self, period: str = "all", metrics: str | Sequence[str] | None = None) -> Self:
+        """Returns filtered result metric data."""
+        if metrics and not isinstance(metrics, (str, Sequence)):
+            raise ValueError("metrics value provided is not a valid metric or sequence of metrics")
+        if isinstance(metrics, str):
+            metrics = [metrics]
+        return self._filter(period, metrics)
+
+    @abstractmethod
+    def _filter(self, period: str, metrics: Sequence[str] | None = None) -> Self: ...
+
+
+class Abstract1DResult(AbstractResult, ABC):
+    def __init__(self, results_data: pd.DataFrame) -> None:
+        super().__init__(results_data)
+
+    def _filter(self, period: str, metrics: Sequence[str] | None = None) -> Self:
+        data = self._data
+        if period != "all":
+            data = self._data.loc[self._data.loc[:, ("chunk", "period")] == period, :]  # type: ignore | dataframe loc
+            data = data.reset_index(drop=True)
+
+        res = copy.deepcopy(self)
+        res._data = data
+        return res
+
+
+class PerMetricResult(Abstract1DResult):
+    def __init__(self, results_data: pd.DataFrame, metrics: Sequence[Metric] = []) -> None:
+        super().__init__(results_data)
+        self.metrics = metrics
+
+    def _filter(self, period: str, metrics: Sequence[str] | None = None) -> Self:
+        if metrics is None:
+            metrics = [metric.column_name for metric in self.metrics]
+
+        res = super()._filter(period)
+
+        data = pd.concat([res._data.loc[:, (["chunk"])], res._data.loc[:, (metrics,)]], axis=1)  # type: ignore | dataframe loc
+        data = data.reset_index(drop=True)
+
+        res._data = data
+        res.metrics = [metric for metric in self.metrics if metric.column_name in metrics]
+
+        return res

dataeval/detectors/drift/_nml/_thresholds.py

@@ -0,0 +1,269 @@
+"""
+Source code derived from NannyML 0.13.0
+https://github.com/NannyML/nannyml/blob/main/nannyml/thresholds.py
+
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Callable, ClassVar
+
+import numpy as np
+
+
+class Threshold(ABC):
+    """A base class used to calculate lower and upper threshold values given one or multiple arrays.
+
+    Any subclass should implement the abstract `thresholds` method.
+    It takes an array or list of arrays and converts them into lower and upper threshold values, represented
+    as a tuple of optional floats.
+
+    A `None` threshold value is interpreted as if there is no upper or lower threshold.
+    One or both values might be `None`.
+    """
+
+    _registry: ClassVar[dict[str, type[Threshold]]] = {}
+    """Class registry lookup to get threshold subclass from threshold_type string"""
+
+    def __str__(self) -> str:
+        return f"{self.__class__.__name__}({str(vars(self))})"
+
+    def __repr__(self) -> str:
+        return str(self)
+
+    def __eq__(self, other: object) -> bool:
+        return isinstance(other, self.__class__) and other.__dict__ == self.__dict__
+
+    def __init_subclass__(cls, threshold_type: str) -> None:
+        Threshold._registry[threshold_type] = cls
+
+    @abstractmethod
+    def _thresholds(self, data: np.ndarray) -> tuple[float | None, float | None]:
+        """Returns lower and upper threshold values when given one or more np.ndarray instances.
+
+        Parameters:
+            data: np.ndarray
+                An array of values used to calculate the thresholds on. This will most often represent a metric
+                calculated on one or more sets of data, e.g. a list of F1 scores of multiple data chunks.
+            kwargs: dict[str, Any]
+                Optional keyword arguments passed to the implementing subclass.
+
+        Returns:
+            lower, upper: tuple[Optional[float], Optional[float]]
+                The lower and upper threshold values. One or both might be `None`.
+        """
+
+    @classmethod
+    def parse_object(cls, obj: dict[str, Any]) -> Threshold:
+        """Parse object as :class:`Threshold`"""
+        class_name = obj.pop("type", "")
+
+        try:
+            threshold_cls = cls._registry[class_name]
+        except KeyError:
+            accepted_values = ", ".join(map(repr, cls._registry))
+            raise ValueError(f"Expected one of {accepted_values} for threshold type, but received '{class_name}'")
+
+        return threshold_cls(**obj)
+
+    def calculate(
+        self,
+        data: np.ndarray,
+        lower_limit: float | None = None,
+        upper_limit: float | None = None,
+        override_using_none: bool = False,
+        logger: logging.Logger | None = None,
+    ) -> tuple[float | None, float | None]:
+        """
+        Calculate lower and upper threshold values with respect to the provided Threshold and value limits.
+
+        Parameters
+        ----------
+        data : np.ndarray
+            The data used by the Threshold instance to calculate the lower and upper threshold values.
+            This will often be the values of a drift detection method or performance metric on chunks of reference
+            data.
+        lower_limit : float or None, default None
+            An optional value that serves as a limit for the lower threshold value. Any calculated lower threshold
+            values that end up below this limit will be replaced by this limit value.
+            The limit is often a theoretical constraint enforced by a specific drift detection method or performance
+            metric.
+        upper_threshold_value_limit : float or None, default None
+            An optional value that serves as a limit for the lower threshold value. Any calculated lower threshold
+            values that end up below this limit will be replaced by this limit value.
+            The limit is often a theoretical constraint enforced by a specific drift detection method or performance
+            metric.
+        override_using_none: bool, default False
+            When set to True use None to override threshold values that exceed value limits.
+            This will prevent them from being rendered on plots.
+        logger: Optional[logging.Logger], default=None
+            An optional Logger instance. When provided a warning will be logged when a calculated threshold value
+            gets overridden by a threshold value limit.
+        """
+
+        lower_value, upper_value = self._thresholds(data)
+
+        if lower_limit is not None and lower_value is not None and lower_value <= lower_limit:
+            override_value = None if override_using_none else lower_limit
+            if logger:
+                logger.warning(
+                    f"lower threshold value {lower_value} overridden by lower threshold value limit {override_value}"
+                )
+            lower_value = override_value
+
+        if upper_limit is not None and upper_value is not None and upper_value >= upper_limit:
+            override_value = None if override_using_none else upper_limit
+            if logger:
+                logger.warning(
+                    f"upper threshold value {upper_value} overridden by upper threshold value limit {override_value}"
+                )
+            upper_value = override_value
+
+        return lower_value, upper_value
+
+
+class ConstantThreshold(Threshold, threshold_type="constant"):
+    """A `Thresholder` implementation that returns a constant lower and or upper threshold value.
+
+    Attributes:
+        lower: Optional[float]
+            The constant lower threshold value. Defaults to `None`, meaning there is no lower threshold.
+        upper: Optional[float]
+            The constant upper threshold value. Defaults to `None`, meaning there is no upper threshold.
+
+    Raises:
+        ValueError: raised when an argument was given using an incorrect type or name
+        ValueError: raised when the ConstantThreshold could not be created using the given argument values
+
+    Examples:
+        >>> data = np.array(range(10))
+        >>> t = ConstantThreshold(lower=None, upper=0.1)
+        >>> lower, upper = t.threshold()
+        >>> print(lower, upper)
+        None 0.1
+    """
+
+    def __init__(self, lower: float | int | None = None, upper: float | int | None = None) -> None:
+        """Creates a new ConstantThreshold instance.
+
+        Args:
+            lower: Optional[Union[float, int]], default=None
+                The constant lower threshold value. Defaults to `None`, meaning there is no lower threshold.
+            upper: Optional[Union[float, int]], default=None
+                The constant upper threshold value. Defaults to `None`, meaning there is no upper threshold.
+
+        Raises:
+            ValueError: raised when an argument was given using an incorrect type or name
+            ValueError: raised when the ConstantThreshold could not be created using the given argument values
+        """
+        self._validate_inputs(lower, upper)
+
+        self.lower = lower
+        self.upper = upper
+
+    def _thresholds(self, data: np.ndarray) -> tuple[float | None, float | None]:
+        return self.lower, self.upper
+
+    @staticmethod
+    def _validate_inputs(lower: float | int | None = None, upper: float | int | None = None) -> None:
+        if lower is not None and not isinstance(lower, (float, int)) or isinstance(lower, bool):
+            raise ValueError(f"expected type of 'lower' to be 'float', 'int' or None but got '{type(lower).__name__}'")
+
+        if upper is not None and not isinstance(upper, (float, int)) or isinstance(upper, bool):
+            raise ValueError(f"expected type of 'upper' to be 'float', 'int' or None but got '{type(upper).__name__}'")
+
+        # explicit None check is required due to special interpretation of the value 0.0 as False
+        if lower is not None and upper is not None and lower >= upper:
+            raise ValueError(f"lower threshold {lower} must be less than upper threshold {upper}")
+
+
+class StandardDeviationThreshold(Threshold, threshold_type="standard_deviation"):
+    """A Thresholder that offsets the mean of an array by a multiple of the standard deviation of the array values.
+
+    This thresholder will take the aggregate of an array of values, the mean by default and add or subtract an offset
+    to get the upper and lower threshold values.
+    This offset is calculated as a multiplier, by default 3, times the standard deviation of the given array.
+
+    Attributes:
+        std_lower_multiplier: float
+        std_upper_multiplier: float
+
+    Examples:
+        >>> data = np.array(range(10))
+        >>> t = ConstantThreshold(lower=None, upper=0.1)
+        >>> lower, upper = t.threshold()
+        >>> print(lower, upper)
+        -4.116843969807043 13.116843969807043
+    """
+
+    def __init__(
+        self,
+        std_lower_multiplier: float | int | None = 3,
+        std_upper_multiplier: float | int | None = 3,
+        offset_from: Callable[[np.ndarray], Any] = np.nanmean,
+    ) -> None:
+        """Creates a new StandardDeviationThreshold instance.
+
+        Args:
+            std_lower_multiplier: float, default=3
+                The number the standard deviation of the input array will be multiplied with to form the lower offset.
+                This value will be subtracted from the aggregate of the input array.
+                Defaults to 3.
+            std_upper_multiplier: float, default=3
+                The number the standard deviation of the input array will be multiplied with to form the upper offset.
+                This value will be added to the aggregate of the input array.
+                Defaults to 3.
+            offset_from: Callable[[np.ndarray], Any], default=np.nanmean
+                A function that will be applied to the input array to aggregate it into a single value.
+                Adding the upper offset to this value will yield the upper threshold, subtracting the lower offset
+                will yield the lower threshold.
+        """
+
+        self._validate_inputs(std_lower_multiplier, std_upper_multiplier)
+
+        self.std_lower_multiplier = std_lower_multiplier
+        self.std_upper_multiplier = std_upper_multiplier
+        self.offset_from = offset_from
+
+    def _thresholds(self, data: np.ndarray) -> tuple[float | None, float | None]:
+        aggregate = self.offset_from(data)
+        std = np.nanstd(data)
+
+        lower_threshold = aggregate - std * self.std_lower_multiplier if self.std_lower_multiplier is not None else None
+
+        upper_threshold = aggregate + std * self.std_upper_multiplier if self.std_upper_multiplier is not None else None
+
+        return lower_threshold, upper_threshold
+
+    @staticmethod
+    def _validate_inputs(
+        std_lower_multiplier: float | int | None = 3, std_upper_multiplier: float | int | None = 3
+    ) -> None:
+        if (
+            std_lower_multiplier is not None
+            and not isinstance(std_lower_multiplier, (float, int))
+            or isinstance(std_lower_multiplier, bool)
+        ):
+            raise ValueError(
+                f"expected type of 'std_lower_multiplier' to be 'float', 'int' or None "
+                f"but got '{type(std_lower_multiplier).__name__}'"
+            )
+
+        if std_lower_multiplier and std_lower_multiplier < 0:
+            raise ValueError(f"'std_lower_multiplier' should be greater than 0 but got value {std_lower_multiplier}")
+
+        if (
+            std_upper_multiplier is not None
+            and not isinstance(std_upper_multiplier, (float, int))
+            or isinstance(std_upper_multiplier, bool)
+        ):
+            raise ValueError(
+                f"expected type of 'std_upper_multiplier' to be 'float', 'int' or None "
+                f"but got '{type(std_upper_multiplier).__name__}'"
+            )
+
+        if std_upper_multiplier and std_upper_multiplier < 0:
+            raise ValueError(f"'std_upper_multiplier' should be greater than 0 but got value {std_upper_multiplier}")

dataeval/detectors/linters/outliers.py

@@ -13,31 +13,31 @@ from dataeval.metrics.stats._imagestats import imagestats
 from dataeval.outputs import DimensionStatsOutput, ImageStatsOutput, OutliersOutput, PixelStatsOutput, VisualStatsOutput
 from dataeval.outputs._base import set_metadata
 from dataeval.outputs._linters import IndexIssueMap, OutlierStatsOutput
-from dataeval.outputs._stats import BOX_COUNT, SOURCE_INDEX
+from dataeval.outputs._stats import BASE_ATTRS
 from dataeval.typing import ArrayLike, Dataset
 
 
 def _get_outlier_mask(
     values: NDArray, method: Literal["zscore", "modzscore", "iqr"], threshold: float | None
 ) -> NDArray:
+    values = values.astype(np.float64)
     if method == "zscore":
         threshold = threshold if threshold else 3.0
         std = np.std(values)
         abs_diff = np.abs(values - np.mean(values))
         return std != 0 and (abs_diff / std) > threshold
-    elif method == "modzscore":
+    if method == "modzscore":
         threshold = threshold if threshold else 3.5
         abs_diff = np.abs(values - np.median(values))
         med_abs_diff = np.median(abs_diff) if np.median(abs_diff) != 0 else np.mean(abs_diff)
         mod_z_score = 0.6745 * abs_diff / med_abs_diff
         return mod_z_score > threshold
-    elif method == "iqr":
+    if method == "iqr":
         threshold = threshold if threshold else 1.5
         qrt = np.percentile(values, q=(25, 75), method="midpoint")
         iqr = (qrt[1] - qrt[0]) * threshold
         return (values < (qrt[0] - iqr)) | (values > (qrt[1] + iqr))
-    else:
-        raise ValueError("Outlier method must be 'zscore' 'modzscore' or 'iqr'.")
+    raise ValueError("Outlier method must be 'zscore' 'modzscore' or 'iqr'.")
 
 
 class Outliers:
@@ -103,7 +103,7 @@ class Outliers:
         use_visual: bool = True,
         outlier_method: Literal["zscore", "modzscore", "iqr"] = "modzscore",
         outlier_threshold: float | None = None,
-    ):
+    ) -> None:
         self.stats: ImageStatsOutput
         self.use_dimension = use_dimension
         self.use_pixel = use_pixel
@@ -114,7 +114,7 @@ class Outliers:
     def _get_outliers(self, stats: dict) -> dict[int, dict[str, float]]:
         flagged_images: dict[int, dict[str, float]] = {}
         for stat, values in stats.items():
-            if stat in (SOURCE_INDEX, BOX_COUNT):
+            if stat in BASE_ATTRS:
                 continue
             if values.ndim == 1:
                 mask = _get_outlier_mask(values.astype(np.float64), self.outlier_method, self.outlier_threshold)

dataeval/metrics/bias/_parity.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 __all__ = []
 
 import warnings
+from collections import defaultdict
 from typing import Any
 
 import numpy as np
@@ -246,7 +247,7 @@ def parity(metadata: Metadata) -> ParityOutput:
 
     chi_scores = np.zeros(metadata.discrete_data.shape[1])
     p_values = np.zeros_like(chi_scores)
-    insufficient_data = {}
+    insufficient_data: defaultdict[str, defaultdict[int, dict[str, int]]] = defaultdict(lambda: defaultdict(dict))
     for i, col_data in enumerate(metadata.discrete_data.T):
         # Builds a contingency matrix where entry at index (r,c) represents
         # the frequency of current_factor_name achieving value unique_factor_values[r]
@@ -261,26 +262,22 @@ def parity(metadata: Metadata) -> ParityOutput:
         for int_factor, int_class in zip(counts[0], counts[1]):
             if contingency_matrix[int_factor, int_class] > 0:
                 factor_category = unique_factor_values[int_factor].item()
-                if current_factor_name not in insufficient_data:
-                    insufficient_data[current_factor_name] = {}
-                if factor_category not in insufficient_data[current_factor_name]:
-                    insufficient_data[current_factor_name][factor_category] = {}
                 class_name = metadata.class_names[int_class]
                 class_count = contingency_matrix[int_factor, int_class].item()
                 insufficient_data[current_factor_name][factor_category][class_name] = class_count
 
         # This deletes rows containing only zeros,
         # because scipy.stats.chi2_contingency fails when there are rows containing only zeros.
-        rowsums = np.sum(contingency_matrix, axis=1)
-        rowmask = np.nonzero(rowsums)[0]
-        contingency_matrix = contingency_matrix[rowmask]
+        contingency_matrix = contingency_matrix[np.any(contingency_matrix, axis=1)]
 
-        chi2, p, _, _ = chi2_contingency(contingency_matrix)
-
-        chi_scores[i] = chi2
-        p_values[i] = p
+        chi_scores[i], p_values[i] = chi2_contingency(contingency_matrix)[:2]
 
     if insufficient_data:
         warnings.warn("Some factors did not meet the recommended 5 occurrences for each value-label combination.")
 
-    return ParityOutput(chi_scores, p_values, metadata.discrete_factor_names, insufficient_data)
+    return ParityOutput(
+        score=chi_scores,
+        p_value=p_values,
+        factor_names=metadata.discrete_factor_names,
+        insufficient_data={k: dict(v) for k, v in insufficient_data.items()},
+    )

dataeval/metrics/estimators/_divergence.py

@@ -38,8 +38,7 @@ def divergence_mst(data: NDArray[np.float64], labels: NDArray[np.int_]) -> int:
     """
     mst = minimum_spanning_tree(data).toarray()
     edgelist = np.transpose(np.nonzero(mst))
-    errors = np.sum(labels[edgelist[:, 0]] != labels[edgelist[:, 1]])
-    return errors
+    return np.sum(labels[edgelist[:, 0]] != labels[edgelist[:, 1]])
 
 
 def divergence_fnn(data: NDArray[np.float64], labels: NDArray[np.int_]) -> int:
@@ -59,8 +58,7 @@ def divergence_fnn(data: NDArray[np.float64], labels: NDArray[np.int_]) -> int:
         Number of label errors when finding nearest neighbors
     """
     nn_indices = compute_neighbors(data, data)
-    errors = np.sum(np.abs(labels[nn_indices] - labels))
-    return errors
+    return np.sum(np.abs(labels[nn_indices] - labels))
 
 
 _DIVERGENCE_FN_MAP = {"FNN": divergence_fnn, "MST": divergence_mst}