dataeval 0.85.0__py3-none-any.whl → 0.86.0__py3-none-any.whl

dataeval/detectors/drift/_nml/_domainclassifier.py

@@ -0,0 +1,192 @@
+"""
+Source code derived from NannyML 0.13.0
+https://github.com/NannyML/nannyml/blob/main/nannyml/drift/multivariate/domain_classifier/calculator.py
+
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from lightgbm import LGBMClassifier
+from numpy.typing import NDArray
+from sklearn.metrics import roc_auc_score
+from sklearn.model_selection import StratifiedKFold
+
+from dataeval.config import get_max_processes, get_seed
+from dataeval.detectors.drift._nml._base import AbstractCalculator, _create_multilevel_index
+from dataeval.detectors.drift._nml._chunk import Chunk, Chunker
+from dataeval.detectors.drift._nml._thresholds import ConstantThreshold, Threshold, calculate_threshold_values
+from dataeval.outputs._base import set_metadata
+from dataeval.outputs._drift import DriftMVDCOutput
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_LGBM_HYPERPARAMS = {
+    "boosting_type": "gbdt",
+    "class_weight": None,
+    "colsample_bytree": 1.0,
+    "deterministic": True,
+    "importance_type": "split",
+    "learning_rate": 0.1,
+    "max_depth": -1,
+    "min_child_samples": 20,
+    "min_child_weight": 0.001,
+    "min_split_gain": 0.0,
+    "n_estimators": 100,
+    "n_jobs": get_max_processes() or 0,
+    "num_leaves": 31,
+    "objective": None,
+    "random_state": get_seed(),
+    "reg_alpha": 0.0,
+    "reg_lambda": 0.0,
+    "subsample": 1.0,
+    "subsample_for_bin": 200000,
+    "subsample_freq": 0,
+    "verbosity": -1,
+}
+
+
+class DomainClassifierCalculator(AbstractCalculator):
+    """
+    DomainClassifierCalculator implementation.
+
+    Uses Drift Detection Classifier's cross validated performance as a measure of drift.
+    """
+
+    def __init__(
+        self,
+        chunker: Chunker | None = None,
+        cv_folds_num: int = 5,
+        hyperparameters: dict[str, Any] | None = None,
+        threshold: Threshold = ConstantThreshold(lower=0.45, upper=0.65),
+    ) -> None:
+        """
+        Create a new DomainClassifierCalculator instance.
+
+        Parameters
+        -----------
+        chunker : Chunker, default=None
+            The `Chunker` used to split the data sets into a lists of chunks.
+        cv_folds_num: Optional[int]
+            Number of cross-validation folds to use when calculating DC discrimination value.
+        hyperparameters : dict[str, Any], default = None
+            A dictionary used to provide your own custom hyperparameters when training the discrimination model.
+            Check out the available hyperparameter options in the
+            `LightGBM docs <https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.LGBMClassifier.html>`_.
+        threshold: Threshold, default=ConstantThreshold
+            The threshold you wish to evaluate values on. Defaults to a ConstantThreshold with lower value
+            of 0.45 and upper value of 0.65.
+        """
+        super().__init__(chunker, logger)
+
+        self.cv_folds_num = cv_folds_num
+        self.hyperparameters = DEFAULT_LGBM_HYPERPARAMS if hyperparameters is None else hyperparameters
+        self.threshold = threshold
+        self.result: DriftMVDCOutput | None = None
+
+    def _fit(self, reference_data: pd.DataFrame) -> DriftMVDCOutput:
+        """Fits the DC calculator to a set of reference data."""
+        self._x_ref = reference_data
+        result = self._calculate(data=self._x_ref)
+        result._data[("chunk", "period")] = "reference"
+
+        return result
+
+    @set_metadata
+    def _calculate(self, data: pd.DataFrame) -> DriftMVDCOutput:
+        """Calculate the data DC calculator metric for a given data set."""
+        chunks = self.chunker.split(data)
+
+        res = pd.DataFrame.from_records(
+            [
+                {
+                    **chunk.dict(),
+                    "period": "analysis",
+                    "classifier_auroc_value": self._calculate_chunk(chunk=chunk),
+                }
+                for chunk in chunks
+            ]
+        )
+
+        multilevel_index = _create_multilevel_index(chunks, "domain_classifier_auroc", ["value"])
+        res.columns = multilevel_index
+        res = res.reset_index(drop=True)
+
+        res = self._populate_alert_thresholds(res)
+
+        if self.result is None:
+            self.result = DriftMVDCOutput(results_data=res)
+        else:
+            self.result = self.result.filter(period="reference")
+            self.result._data = pd.concat([self.result._data, res], ignore_index=True)
+        return self.result
+
+    def _calculate_chunk(self, chunk: Chunk):
+        if self.result is None:
+            # Use information from chunk indices to identify reference chunk's location. This is possible because
+            # both the internal reference data copy and the chunk data were sorted by timestamp, so these
+            # indices align. This way we eliminate the need to combine these two data frames and drop duplicate rows,
+            # which is a costly operation.
+            df_X = self._x_ref
+            y = np.zeros(len(df_X), dtype=np.intp)
+            y[chunk.start_index : chunk.end_index + 1] = 1
+        else:
+            chunk_X = chunk.data
+            reference_X = self._x_ref
+            chunk_y = np.ones(len(chunk_X), dtype=np.intp)
+            reference_y = np.zeros(len(reference_X), dtype=np.intp)
+            df_X = pd.concat([reference_X, chunk_X], ignore_index=True)
+            y = np.concatenate([reference_y, chunk_y])
+
+        skf = StratifiedKFold(n_splits=self.cv_folds_num)
+        all_preds: list[NDArray[np.float32]] = []
+        all_tgts: list[NDArray[np.intp]] = []
+        for i, (train_index, test_index) in enumerate(skf.split(df_X, y)):
+            _trx = df_X.iloc[train_index]
+            _try = y[train_index]
+            _tsx = df_X.iloc[test_index]
+            _tsy = y[test_index]
+            model = LGBMClassifier(**self.hyperparameters)
+            model.fit(_trx, _try)
+            preds = np.asarray(model.predict_proba(_tsx), dtype=np.float32)[:, 1]
+            all_preds.append(preds)
+            all_tgts.append(_tsy)
+
+        np_all_preds = np.concatenate(all_preds, axis=0)
+        np_all_tgts = np.concatenate(all_tgts, axis=0)
+        try:
+            # catch case where all rows are duplicates
+            result = roc_auc_score(np_all_tgts, np_all_preds)
+        except ValueError as err:
+            if str(err) != "Only one class present in y_true. ROC AUC score is not defined in that case.":
+                raise
+            else:
+                # by definition if reference and chunk exactly match we can't discriminate
+                result = 0.5
+        return result
+
+    def _populate_alert_thresholds(self, result_data: pd.DataFrame) -> pd.DataFrame:
+        if self.result is None:
+            self._threshold_values = calculate_threshold_values(
+                threshold=self.threshold,
+                data=result_data.loc[:, ("domain_classifier_auroc", "value")],  # type: ignore | dataframe loc
+                lower_threshold_value_limit=0.0,
+                upper_threshold_value_limit=1.0,
+                logger=self._logger,
+            )
+
+        result_data[("domain_classifier_auroc", "upper_threshold")] = self._threshold_values[1]
+        result_data[("domain_classifier_auroc", "lower_threshold")] = self._threshold_values[0]
+        result_data[("domain_classifier_auroc", "alert")] = result_data.apply(
+            lambda row: bool(
+                row["domain_classifier_auroc", "value"] > row["domain_classifier_auroc", "upper_threshold"]
+                or row["domain_classifier_auroc", "value"] < row["domain_classifier_auroc", "lower_threshold"]
+            ),
+            axis=1,
+        )
+        return result_data

dataeval/detectors/drift/_nml/_result.py

@@ -0,0 +1,98 @@
+"""
+Contains the results of the data reconstruction drift calculation and provides plotting functionality.
+
+Source code derived from NannyML 0.13.0
+https://github.com/NannyML/nannyml/blob/main/nannyml/base.py
+
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from __future__ import annotations
+
+import copy
+from abc import ABC, abstractmethod
+from typing import NamedTuple, Sequence
+
+import pandas as pd
+from typing_extensions import Self
+
+from dataeval.outputs._base import GenericOutput
+
+
+class Metric(NamedTuple):
+    display_name: str
+    column_name: str
+
+
+class AbstractResult(GenericOutput[pd.DataFrame]):
+    def __init__(self, results_data: pd.DataFrame) -> None:
+        self._data = results_data.copy(deep=True)
+
+    def data(self) -> pd.DataFrame:
+        return self.to_df()
+
+    @property
+    def empty(self) -> bool:
+        return self._data is None or self._data.empty
+
+    def __len__(self) -> int:
+        return 0 if self.empty else len(self._data)
+
+    def to_df(self, multilevel: bool = True) -> pd.DataFrame:
+        """Export results to pandas dataframe."""
+        if multilevel:
+            return self._data
+        else:
+            column_names = [
+                "_".join(col).replace("chunk_chunk_chunk", "chunk").replace("chunk_chunk", "chunk")
+                for col in self._data.columns.values
+            ]
+            single_level_data = self._data.copy(deep=True)
+            single_level_data.columns = column_names
+            return single_level_data
+
+    def filter(self, period: str = "all", metrics: str | Sequence[str] | None = None) -> Self:
+        """Returns filtered result metric data."""
+        if metrics and not isinstance(metrics, (str, Sequence)):
+            raise ValueError("metrics value provided is not a valid metric or sequence of metrics")
+        if isinstance(metrics, str):
+            metrics = [metrics]
+        return self._filter(period, metrics)
+
+    @abstractmethod
+    def _filter(self, period: str, metrics: Sequence[str] | None = None) -> Self: ...
+
+
+class Abstract1DResult(AbstractResult, ABC):
+    def __init__(self, results_data: pd.DataFrame) -> None:
+        super().__init__(results_data)
+
+    def _filter(self, period: str, metrics=None) -> Self:
+        data = self._data
+        if period != "all":
+            data = self._data.loc[self._data.loc[:, ("chunk", "period")] == period, :]  # type: ignore | dataframe loc
+            data = data.reset_index(drop=True)
+
+        res = copy.deepcopy(self)
+        res._data = data
+        return res
+
+
+class PerMetricResult(Abstract1DResult):
+    def __init__(self, results_data: pd.DataFrame, metrics: Sequence[Metric] = []) -> None:
+        super().__init__(results_data)
+        self.metrics = metrics
+
+    def _filter(self, period: str, metrics: Sequence[str] | None = None) -> Self:
+        if metrics is None:
+            metrics = [metric.column_name for metric in self.metrics]
+
+        res = super()._filter(period)
+
+        data = pd.concat([res._data.loc[:, (["chunk"])], res._data.loc[:, (metrics,)]], axis=1)  # type: ignore | dataframe loc
+        data = data.reset_index(drop=True)
+
+        res._data = data
+        res.metrics = [metric for metric in self.metrics if metric.column_name in metrics]
+
+        return res

dataeval/detectors/drift/_nml/_thresholds.py

@@ -0,0 +1,280 @@
+"""
+Source code derived from NannyML 0.13.0
+https://github.com/NannyML/nannyml/blob/main/nannyml/thresholds.py
+
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Callable, ClassVar
+
+import numpy as np
+
+
+class Threshold(ABC):
+    """A base class used to calculate lower and upper threshold values given one or multiple arrays.
+
+    Any subclass should implement the abstract `thresholds` method.
+    It takes an array or list of arrays and converts them into lower and upper threshold values, represented
+    as a tuple of optional floats.
+
+    A `None` threshold value is interpreted as if there is no upper or lower threshold.
+    One or both values might be `None`.
+    """
+
+    _registry: ClassVar[dict[str, type[Threshold]]] = {}
+    """Class registry lookup to get threshold subclass from threshold_type string"""
+
+    def __str__(self) -> str:
+        return self.__str__()
+
+    def __repr__(self) -> str:
+        return self.__class__.__name__ + str(vars(self))
+
+    def __eq__(self, other: object) -> bool:
+        return isinstance(other, self.__class__) and other.__dict__ == self.__dict__
+
+    def __init_subclass__(cls, threshold_type: str) -> None:
+        Threshold._registry[threshold_type] = cls
+
+    @abstractmethod
+    def thresholds(self, data: np.ndarray) -> tuple[float | None, float | None]:
+        """Returns lower and upper threshold values when given one or more np.ndarray instances.
+
+        Parameters:
+            data: np.ndarray
+                An array of values used to calculate the thresholds on. This will most often represent a metric
+                calculated on one or more sets of data, e.g. a list of F1 scores of multiple data chunks.
+            kwargs: dict[str, Any]
+                Optional keyword arguments passed to the implementing subclass.
+
+        Returns:
+            lower, upper: tuple[Optional[float], Optional[float]]
+                The lower and upper threshold values. One or both might be `None`.
+        """
+
+    @classmethod
+    def parse_object(cls, obj: dict[str, Any]) -> Threshold:
+        """Parse object as :class:`Threshold`"""
+        class_name = obj.pop("type", "")
+
+        try:
+            threshold_cls = cls._registry[class_name]
+        except KeyError:
+            accepted_values = ", ".join(map(repr, cls._registry))
+            raise ValueError(f"Expected one of {accepted_values} for threshold type, but received '{class_name}'")
+
+        return threshold_cls(**obj)
+
+
+class ConstantThreshold(Threshold, threshold_type="constant"):
+    """A `Thresholder` implementation that returns a constant lower and or upper threshold value.
+
+    Attributes:
+        lower: Optional[float]
+            The constant lower threshold value. Defaults to `None`, meaning there is no lower threshold.
+        upper: Optional[float]
+            The constant upper threshold value. Defaults to `None`, meaning there is no upper threshold.
+
+    Raises:
+        ValueError: raised when an argument was given using an incorrect type or name
+        ValueError: raised when the ConstantThreshold could not be created using the given argument values
+
+    Examples:
+        >>> data = np.array(range(10))
+        >>> t = ConstantThreshold(lower=None, upper=0.1)
+        >>> lower, upper = t.threshold()
+        >>> print(lower, upper)
+        None 0.1
+    """
+
+    def __init__(self, lower: float | int | None = None, upper: float | int | None = None):
+        """Creates a new ConstantThreshold instance.
+
+        Args:
+            lower: Optional[Union[float, int]], default=None
+                The constant lower threshold value. Defaults to `None`, meaning there is no lower threshold.
+            upper: Optional[Union[float, int]], default=None
+                The constant upper threshold value. Defaults to `None`, meaning there is no upper threshold.
+
+        Raises:
+            ValueError: raised when an argument was given using an incorrect type or name
+            ValueError: raised when the ConstantThreshold could not be created using the given argument values
+        """
+        self._validate_inputs(lower, upper)
+
+        self.lower = lower
+        self.upper = upper
+
+    def thresholds(self, data: np.ndarray) -> tuple[float | None, float | None]:
+        return self.lower, self.upper
+
+    @staticmethod
+    def _validate_inputs(lower: float | int | None = None, upper: float | int | None = None):
+        if lower is not None and not isinstance(lower, (float, int)) or isinstance(lower, bool):
+            raise ValueError(f"expected type of 'lower' to be 'float', 'int' or None but got '{type(lower).__name__}'")
+
+        if upper is not None and not isinstance(upper, (float, int)) or isinstance(upper, bool):
+            raise ValueError(f"expected type of 'upper' to be 'float', 'int' or None but got '{type(upper).__name__}'")
+
+        # explicit None check is required due to special interpretation of the value 0.0 as False
+        if lower is not None and upper is not None and lower >= upper:
+            raise ValueError(f"lower threshold {lower} must be less than upper threshold {upper}")
+
+
+class StandardDeviationThreshold(Threshold, threshold_type="standard_deviation"):
+    """A Thresholder that offsets the mean of an array by a multiple of the standard deviation of the array values.
+
+    This thresholder will take the aggregate of an array of values, the mean by default and add or subtract an offset
+    to get the upper and lower threshold values.
+    This offset is calculated as a multiplier, by default 3, times the standard deviation of the given array.
+
+    Attributes:
+        std_lower_multiplier: float
+        std_upper_multiplier: float
+
+    Examples:
+        >>> data = np.array(range(10))
+        >>> t = ConstantThreshold(lower=None, upper=0.1)
+        >>> lower, upper = t.threshold()
+        >>> print(lower, upper)
+        -4.116843969807043 13.116843969807043
+    """
+
+    def __init__(
+        self,
+        std_lower_multiplier: float | int | None = 3,
+        std_upper_multiplier: float | int | None = 3,
+        offset_from: Callable[[np.ndarray], Any] = np.nanmean,
+    ):
+        """Creates a new StandardDeviationThreshold instance.
+
+        Args:
+            std_lower_multiplier: float, default=3
+                The number the standard deviation of the input array will be multiplied with to form the lower offset.
+                This value will be subtracted from the aggregate of the input array.
+                Defaults to 3.
+            std_upper_multiplier: float, default=3
+                The number the standard deviation of the input array will be multiplied with to form the upper offset.
+                This value will be added to the aggregate of the input array.
+                Defaults to 3.
+            offset_from: Callable[[np.ndarray], Any], default=np.nanmean
+                A function that will be applied to the input array to aggregate it into a single value.
+                Adding the upper offset to this value will yield the upper threshold, subtracting the lower offset
+                will yield the lower threshold.
+        """
+
+        self._validate_inputs(std_lower_multiplier, std_upper_multiplier)
+
+        self.std_lower_multiplier = std_lower_multiplier
+        self.std_upper_multiplier = std_upper_multiplier
+        self.offset_from = offset_from
+
+    def thresholds(self, data: np.ndarray) -> tuple[float | None, float | None]:
+        aggregate = self.offset_from(data)
+        std = np.nanstd(data)
+
+        lower_threshold = aggregate - std * self.std_lower_multiplier if self.std_lower_multiplier is not None else None
+
+        upper_threshold = aggregate + std * self.std_upper_multiplier if self.std_upper_multiplier is not None else None
+
+        return lower_threshold, upper_threshold
+
+    @staticmethod
+    def _validate_inputs(std_lower_multiplier: float | int | None = 3, std_upper_multiplier: float | int | None = 3):
+        if (
+            std_lower_multiplier is not None
+            and not isinstance(std_lower_multiplier, (float, int))
+            or isinstance(std_lower_multiplier, bool)
+        ):
+            raise ValueError(
+                f"expected type of 'std_lower_multiplier' to be 'float', 'int' or None "
+                f"but got '{type(std_lower_multiplier).__name__}'"
+            )
+
+        if std_lower_multiplier and std_lower_multiplier < 0:
+            raise ValueError(f"'std_lower_multiplier' should be greater than 0 but got value {std_lower_multiplier}")
+
+        if (
+            std_upper_multiplier is not None
+            and not isinstance(std_upper_multiplier, (float, int))
+            or isinstance(std_upper_multiplier, bool)
+        ):
+            raise ValueError(
+                f"expected type of 'std_upper_multiplier' to be 'float', 'int' or None "
+                f"but got '{type(std_upper_multiplier).__name__}'"
+            )
+
+        if std_upper_multiplier and std_upper_multiplier < 0:
+            raise ValueError(f"'std_upper_multiplier' should be greater than 0 but got value {std_upper_multiplier}")
+
+
+def calculate_threshold_values(
+    threshold: Threshold,
+    data: np.ndarray,
+    lower_threshold_value_limit: float | None = None,
+    upper_threshold_value_limit: float | None = None,
+    override_using_none: bool = False,
+    logger: logging.Logger | None = None,
+    metric_name: str | None = None,
+) -> tuple[float | None, float | None]:
+    """Calculate lower and upper threshold values with respect to the provided Threshold and value limits.
+
+    Parameters:
+        threshold: Threshold
+            The Threshold instance that determines how the lower and upper threshold values will be calculated.
+        data: np.ndarray
+            The data used by the Threshold instance to calculate the lower and upper threshold values.
+            This will often be the values of a drift detection method or performance metric on chunks of reference data.
+        lower_threshold_value_limit: Optional[float], default=None
+            An optional value that serves as a limit for the lower threshold value. Any calculated lower threshold
+            values that end up below this limit will be replaced by this limit value.
+            The limit is often a theoretical constraint enforced by a specific drift detection method or performance
+            metric.
+        upper_threshold_value_limit: Optional[float], default=None
+            An optional value that serves as a limit for the lower threshold value. Any calculated lower threshold
+            values that end up below this limit will be replaced by this limit value.
+            The limit is often a theoretical constraint enforced by a specific drift detection method or performance
+            metric.
+        override_using_none: bool, default=False
+            When set to True use None to override threshold values that exceed value limits.
+            This will prevent them from being rendered on plots.
+        logger: Optional[logging.Logger], default=None
+            An optional Logger instance. When provided a warning will be logged when a calculated threshold value
+            gets overridden by a threshold value limit.
+        metric_name: Optional[str], default=None
+            When provided the metric name will be included within any log messages for additional clarity.
+    """
+
+    lower_threshold_value, upper_threshold_value = threshold.thresholds(data)
+
+    if (
+        lower_threshold_value_limit is not None
+        and lower_threshold_value is not None
+        and lower_threshold_value <= lower_threshold_value_limit
+    ):
+        override_value = None if override_using_none else lower_threshold_value_limit
+        if logger:
+            logger.warning(
+                f"{metric_name + ' ' if metric_name else ''}lower threshold value {lower_threshold_value} "
+                f"overridden by lower threshold value limit {override_value}"
+            )
+        lower_threshold_value = override_value
+
+    if (
+        upper_threshold_value_limit is not None
+        and upper_threshold_value is not None
+        and upper_threshold_value >= upper_threshold_value_limit
+    ):
+        override_value = None if override_using_none else upper_threshold_value_limit
+        if logger:
+            logger.warning(
+                f"{metric_name + ' ' if metric_name else ''}upper threshold value {upper_threshold_value} "
+                f"overridden by upper threshold value limit {override_value}"
+            )
+        upper_threshold_value = override_value
+
+    return lower_threshold_value, upper_threshold_value

dataeval/outputs/__init__.py

@@ -5,7 +5,7 @@ as well as runtime metadata for reproducibility and logging.
 
 from ._base import ExecutionMetadata
 from ._bias import BalanceOutput, CompletenessOutput, CoverageOutput, DiversityOutput, LabelParityOutput, ParityOutput
-from ._drift import DriftMMDOutput, DriftOutput
+from ._drift import DriftMMDOutput, DriftMVDCOutput, DriftOutput
 from ._estimators import BEROutput, ClustererOutput, DivergenceOutput, UAPOutput
 from ._linters import DuplicatesOutput, OutliersOutput
 from ._metadata import MetadataDistanceOutput, MetadataDistanceValues, MostDeviatedFactorsOutput, OODPredictorOutput
@@ -34,6 +34,7 @@ __all__ = [
     "DivergenceOutput",
     "DiversityOutput",
     "DriftMMDOutput",
+    "DriftMVDCOutput",
     "DriftOutput",
     "DuplicatesOutput",
     "ExecutionMetadata",

dataeval/outputs/_bias.py

@@ -7,10 +7,10 @@ from dataclasses import asdict, dataclass
 from typing import Any, Literal, TypeVar, overload
 
 import numpy as np
+import pandas as pd
 from numpy.typing import NDArray
 
 with contextlib.suppress(ImportError):
-    import pandas as pd
     from matplotlib.figure import Figure
 
 from dataeval.data._images import Images
@@ -38,8 +38,6 @@ class ToDataFrameMixin:
         -----
         This method requires `pandas <https://pandas.pydata.org/>`_ to be installed.
         """
-        import pandas as pd
-
         return pd.DataFrame(
             index=self.factor_names,  # type: ignore - list[str] is documented as acceptable index type
             data={