dataeval 0.85.0__py3-none-any.whl → 0.86.1__py3-none-any.whl

dataeval/detectors/drift/_nml/_base.py

@@ -0,0 +1,70 @@
+"""
+Source code derived from NannyML 0.13.0
+https://github.com/NannyML/nannyml/blob/main/nannyml/base.py
+
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from logging import Logger
+from typing import Sequence
+
+import pandas as pd
+from typing_extensions import Self
+
+from dataeval.detectors.drift._nml._chunk import Chunk, Chunker, CountBasedChunker
+from dataeval.outputs._drift import DriftMVDCOutput
+
+
+def _validate(data: pd.DataFrame, expected_features: int | None = None) -> int:
+    if data.empty:
+        raise ValueError("data contains no rows. Please provide a valid data set.")
+    if expected_features is not None and data.shape[-1] != expected_features:
+        raise ValueError(f"expected '{expected_features}' features in data set:\n\t{data}")
+    return data.shape[-1]
+
+
+def _create_multilevel_index(
+    chunks: Sequence[Chunk], result_group_name: str, result_column_names: Sequence[str]
+) -> pd.MultiIndex:
+    chunk_column_names = (*chunks[0].KEYS, "period")
+    chunk_tuples = [("chunk", chunk_column_name) for chunk_column_name in chunk_column_names]
+    result_tuples = [(result_group_name, column_name) for column_name in result_column_names]
+    return pd.MultiIndex.from_tuples(chunk_tuples + result_tuples)
+
+
+class AbstractCalculator(ABC):
+    """Base class for drift calculation."""
+
+    def __init__(self, chunker: Chunker | None = None, logger: Logger | None = None) -> None:
+        self.chunker = chunker if isinstance(chunker, Chunker) else CountBasedChunker(10)
+        self.result: DriftMVDCOutput | None = None
+        self.n_features: int | None = None
+        self._logger = logger if isinstance(logger, Logger) else logging.getLogger(__name__)
+
+    def fit(self, reference_data: pd.DataFrame) -> Self:
+        """Trains the calculator using reference data."""
+        self.n_features = _validate(reference_data)
+
+        self._logger.debug(f"fitting {str(self)}")
+        self.result = self._fit(reference_data)
+        return self
+
+    def calculate(self, data: pd.DataFrame) -> DriftMVDCOutput:
+        """Performs a calculation on the provided data."""
+        if self.result is None:
+            raise RuntimeError("must run fit with reference data before running calculate")
+        _validate(data, self.n_features)
+
+        self._logger.debug(f"calculating {str(self)}")
+        self.result = self._calculate(data)
+        return self.result
+
+    @abstractmethod
+    def _fit(self, reference_data: pd.DataFrame) -> DriftMVDCOutput: ...
+
+    @abstractmethod
+    def _calculate(self, data: pd.DataFrame) -> DriftMVDCOutput: ...

dataeval/detectors/drift/_nml/_chunk.py

@@ -0,0 +1,396 @@
+"""
+NannyML module providing intelligent splitting of data into chunks.
+
+Source code derived from NannyML 0.13.0
+https://github.com/NannyML/nannyml/blob/main/nannyml/chunk.py
+
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from __future__ import annotations
+
+import copy
+import logging
+import warnings
+from abc import ABC, abstractmethod
+from typing import Any, Generic, Literal, Sequence, TypeVar, cast
+
+import pandas as pd
+from pandas import Index, Period
+from typing_extensions import Self
+
+logger = logging.getLogger(__name__)
+
+
+class Chunk(ABC):
+    """A subset of data that acts as a logical unit during calculations."""
+
+    KEYS: Sequence[str]
+
+    def __init__(
+        self,
+        data: pd.DataFrame,
+    ) -> None:
+        self.key: str
+        self.data = data
+
+        self.start_index: int = -1
+        self.end_index: int = -1
+        self.chunk_index: int = -1
+
+    def __repr__(self) -> str:
+        attr_str = ", ".join([f"{k}={v}" for k, v in self.dict().items()])
+        return f"{self.__class__.__name__}(data=pd.DataFrame(shape={self.data.shape}), {attr_str})"
+
+    def __len__(self) -> int:
+        return self.data.shape[0]
+
+    @abstractmethod
+    def __add__(self, other: Self) -> Self: ...
+
+    @abstractmethod
+    def __lt__(self, other: Self) -> bool: ...
+
+    @abstractmethod
+    def dict(self) -> dict[str, Any]: ...
+
+
+class IndexChunk(Chunk):
+    """Creates a new chunk.
+
+    Parameters
+    ----------
+    data : DataFrame, required
+        The data to be contained within the chunk.
+    start_datetime: datetime
+        The starting point in time for this chunk.
+    end_datetime: datetime
+        The end point in time for this chunk.
+    """
+
+    KEYS = ("key", "chunk_index", "start_index", "end_index")
+
+    def __init__(
+        self,
+        data: pd.DataFrame,
+        start_index: int,
+        end_index: int,
+    ) -> None:
+        super().__init__(data)
+        self.key = f"[{start_index}:{end_index}]"
+        self.start_index: int = start_index
+        self.end_index: int = end_index
+
+    def __lt__(self, other: Self) -> bool:
+        return self.end_index < other.start_index
+
+    def __add__(self, other: Self) -> Self:
+        a, b = (self, other) if self < other else (other, self)
+        result = copy.deepcopy(a)
+        result.data = pd.concat([a.data, b.data])
+        result.end_index = b.end_index
+        return result
+
+    def dict(self) -> dict[str, Any]:
+        return dict(zip(self.KEYS, (self.key, self.chunk_index, self.start_index, self.end_index)))
+
+
+class PeriodChunk(Chunk):
+    """Creates a new chunk.
+
+    Parameters
+    ----------
+    data : DataFrame, required
+        The data to be contained within the chunk.
+    start_datetime: datetime
+        The starting point in time for this chunk.
+    end_datetime: datetime
+        The end point in time for this chunk.
+    chunk_size : int
+        The size of the chunk.
+    """
+
+    KEYS = ("key", "chunk_index", "start_date", "end_date", "chunk_size")
+
+    def __init__(self, data: pd.DataFrame, period: Period, chunk_size: int) -> None:
+        super().__init__(data)
+        self.key = str(period)
+        self.start_datetime = period.start_time
+        self.end_datetime = period.end_time
+        self.chunk_size = chunk_size
+
+    def __lt__(self, other: Self) -> bool:
+        return self.end_datetime < other.start_datetime
+
+    def __add__(self, other: Self) -> Self:
+        a, b = (self, other) if self < other else (other, self)
+        result = copy.deepcopy(a)
+        result.data = pd.concat([a.data, b.data])
+        result.end_index = b.end_index
+        result.end_datetime = b.end_datetime
+        result.chunk_size += b.chunk_size
+        return result
+
+    def dict(self) -> dict[str, Any]:
+        return dict(
+            zip(self.KEYS, (self.key, self.chunk_index, self.start_datetime, self.end_datetime, self.chunk_size))
+        )
+
+
+TChunk = TypeVar("TChunk", bound=Chunk)
+
+
+class Chunker(Generic[TChunk]):
+    """Base class for Chunker implementations.
+
+    Inheriting classes will split a DataFrame into a list of Chunks.
+    They will do this based on several constraints, e.g. observation timestamps, number of observations per Chunk
+    or a preferred number of Chunks.
+    """
+
+    def split(self, data: pd.DataFrame) -> list[TChunk]:
+        """Splits a given data frame into a list of chunks.
+
+        This method provides a uniform interface across Chunker implementations to keep them interchangeable.
+
+        After performing the implementation-specific `_split` method, there are some checks on the resulting chunk list.
+
+        If the total number of chunks is low a warning will be written out to the logs.
+
+        We dynamically determine the optimal minimum number of observations per chunk and then check if the resulting
+        chunks contain at least as many. If there are any underpopulated chunks a warning will be written out in
+        the logs.
+
+        Parameters
+        ----------
+        data: DataFrame
+            The data to be split into chunks
+
+        Returns
+        -------
+        chunks: List[Chunk]
+            The list of chunks
+
+        """
+        if data.shape[0] == 0:
+            return []
+
+        chunks = self._split(data)
+        for chunk_index, chunk in enumerate(chunks):
+            chunk.start_index = cast(int, chunk.data.index.min())
+            chunk.end_index = cast(int, chunk.data.index.max())
+            chunk.chunk_index = chunk_index
+
+        if len(chunks) < 6:
+            # TODO wording
+            warnings.warn(
+                "The resulting number of chunks is too low. "
+                "Please consider splitting your data in a different way or continue at your own risk."
+            )
+
+        return chunks
+
+    @abstractmethod
+    def _split(self, data: pd.DataFrame) -> list[TChunk]: ...
+
+
+class PeriodBasedChunker(Chunker[PeriodChunk]):
+    """A Chunker that will split data into Chunks based on a date column in the data.
+
+    Examples
+    --------
+    Chunk using monthly periods and providing a column name
+
+    >>> from nannyml.chunk import PeriodBasedChunker
+    >>> df = pd.read_parquet("/path/to/my/data.pq")
+    >>> chunker = PeriodBasedChunker(timestamp_column_name="observation_date", offset="M")
+    >>> chunks = chunker.split(data=df)
+
+    Or chunk using weekly periods
+
+    >>> from nannyml.chunk import PeriodBasedChunker
+    >>> df = pd.read_parquet("/path/to/my/data.pq")
+    >>> chunker = PeriodBasedChunker(timestamp_column_name=df["observation_date"], offset="W", minimum_chunk_size=50)
+    >>> chunks = chunker.split(data=df)
+
+    """
+
+    def __init__(self, timestamp_column_name: str, offset: str = "W") -> None:
+        """Creates a new PeriodBasedChunker.
+
+        Parameters
+        ----------
+        timestamp_column_name : str
+            The column name containing the timestamp to chunk on
+        offset : str
+            A frequency string representing a pandas.tseries.offsets.DateOffset.
+            The offset determines how the time-based grouping will occur. A list of possible values
+            can be found at <https://pandas.pydata.org/docs/user_guide/timeseries.html#offset-aliases>.
+        """
+        self.timestamp_column_name = timestamp_column_name
+        self.offset = offset
+
+    def _split(self, data: pd.DataFrame) -> list[PeriodChunk]:
+        chunks = []
+        if self.timestamp_column_name is None:
+            raise ValueError("timestamp_column_name must be provided")
+        if self.timestamp_column_name not in data:
+            raise ValueError(f"timestamp column '{self.timestamp_column_name}' not in columns")
+
+        grouped = data.groupby(pd.to_datetime(data[self.timestamp_column_name]).dt.to_period(self.offset))
+
+        for k, v in grouped.groups.items():
+            period, index = cast(Period, k), cast(Index, v)
+            chunk = PeriodChunk(
+                data=grouped.get_group(period),  # type: ignore | dataframe
+                period=period,
+                chunk_size=len(index),
+            )
+            chunks.append(chunk)
+
+        return chunks
+
+
+class SizeBasedChunker(Chunker[IndexChunk]):
+    """A Chunker that will split data into Chunks based on the preferred number of observations per Chunk.
+
+    Notes
+    -----
+    - Chunks are adjacent, not overlapping
+    - There may be "incomplete" chunks, as the remainder of observations after dividing by `chunk_size`
+      will form a chunk of their own.
+
+    Examples
+    --------
+    Chunk using monthly periods and providing a column name
+
+    >>> from nannyml.chunk import SizeBasedChunker
+    >>> df = pd.read_parquet("/path/to/my/data.pq")
+    >>> chunker = SizeBasedChunker(chunk_size=2000, incomplete="drop")
+    >>> chunks = chunker.split(data=df)
+
+    """
+
+    def __init__(
+        self,
+        chunk_size: int,
+        incomplete: Literal["append", "drop", "keep"] = "keep",
+    ) -> None:
+        """Create a new SizeBasedChunker.
+
+        Parameters
+        ----------
+        chunk_size: int
+            The preferred size of the resulting Chunks, i.e. the number of observations in each Chunk.
+        incomplete: str, default='keep'
+            Choose how to handle any leftover observations that don't make up a full Chunk.
+            The following options are available:
+
+            - ``'drop'``: drop the leftover observations
+            - ``'keep'``: keep the incomplete Chunk (containing less than ``chunk_size`` observations)
+            - ``'append'``: append leftover observations to the last complete Chunk (overfilling it)
+
+            Defaults to ``'keep'``.
+
+        Returns
+        -------
+        chunker: a size-based instance used to split data into Chunks of a constant size.
+
+        """
+        if not isinstance(chunk_size, int) or chunk_size <= 0:
+            raise ValueError(f"chunk_size={chunk_size} is invalid - provide an integer greater than 0")
+        if incomplete not in ("append", "drop", "keep"):
+            raise ValueError(f"incomplete={incomplete} is invalid - must be one of ['append', 'drop', 'keep']")
+
+        self.chunk_size = chunk_size
+        self.incomplete = incomplete
+
+    def _split(self, data: pd.DataFrame) -> list[IndexChunk]:
+        def _create_chunk(index: int, data: pd.DataFrame, chunk_size: int) -> IndexChunk:
+            chunk_data = data.iloc[index : index + chunk_size]
+            return IndexChunk(
+                data=chunk_data,
+                start_index=index,
+                end_index=index + chunk_size - 1,
+            )
+
+        chunks = [
+            _create_chunk(index=i, data=data, chunk_size=self.chunk_size)
+            for i in range(0, data.shape[0], self.chunk_size)
+            if i + self.chunk_size - 1 < len(data)
+        ]
+
+        # deal with unassigned observations
+        if data.shape[0] % self.chunk_size != 0 and self.incomplete != "drop":
+            incomplete_chunk = _create_chunk(
+                index=self.chunk_size * (data.shape[0] // self.chunk_size),
+                data=data,
+                chunk_size=(data.shape[0] % self.chunk_size),
+            )
+            if self.incomplete == "append":
+                chunks[-1] += incomplete_chunk
+            else:
+                chunks += [incomplete_chunk]
+
+        return chunks
+
+
+class CountBasedChunker(Chunker[IndexChunk]):
+    """A Chunker that will split data into chunks based on the preferred number of total chunks.
+
+    Notes
+    -----
+    - Chunks are adjacent, not overlapping
+    - There may be "incomplete" chunks, as the remainder of observations after dividing by `chunk_size`
+      will form a chunk of their own.
+
+    Examples
+    --------
+    >>> from nannyml.chunk import CountBasedChunker
+    >>> df = pd.read_parquet("/path/to/my/data.pq")
+    >>> chunker = CountBasedChunker(chunk_number=100)
+    >>> chunks = chunker.split(data=df)
+
+    """
+
+    def __init__(
+        self,
+        chunk_number: int,
+        incomplete: Literal["append", "drop", "keep"] = "keep",
+    ) -> None:
+        """Creates a new CountBasedChunker.
+
+        It will calculate the amount of observations per chunk based on the given chunk count.
+        It then continues to split the data into chunks just like a SizeBasedChunker does.
+
+        Parameters
+        ----------
+        chunk_number: int
+            The amount of chunks to split the data in.
+        incomplete: str, default='keep'
+            Choose how to handle any leftover observations that don't make up a full Chunk.
+            The following options are available:
+
+            - ``'drop'``: drop the leftover observations
+            - ``'keep'``: keep the incomplete Chunk (containing less than ``chunk_size`` observations)
+            - ``'append'``: append leftover observations to the last complete Chunk (overfilling it)
+
+            Defaults to ``'keep'``.
+
+        Returns
+        -------
+        chunker: CountBasedChunker
+
+        """
+        if not isinstance(chunk_number, int) or chunk_number <= 0:
+            raise ValueError(f"given chunk_number {chunk_number} is invalid - provide an integer greater than 0")
+        if incomplete not in ("append", "drop", "keep"):
+            raise ValueError(f"incomplete={incomplete} is invalid - must be one of ['append', 'drop', 'keep']")
+
+        self.chunk_number = chunk_number
+        self.incomplete: Literal["append", "drop", "keep"] = incomplete
+
+    def _split(self, data: pd.DataFrame) -> list[IndexChunk]:
+        chunk_size = data.shape[0] // self.chunk_number
+        chunker = SizeBasedChunker(chunk_size, self.incomplete)
+        return chunker.split(data=data)

dataeval/detectors/drift/_nml/_domainclassifier.py

@@ -0,0 +1,181 @@
+"""
+Source code derived from NannyML 0.13.0
+https://github.com/NannyML/nannyml/blob/main/nannyml/drift/multivariate/domain_classifier/calculator.py
+
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from lightgbm import LGBMClassifier
+from numpy.typing import NDArray
+from sklearn.metrics import roc_auc_score
+from sklearn.model_selection import StratifiedKFold
+
+from dataeval.config import get_max_processes, get_seed
+from dataeval.detectors.drift._nml._base import AbstractCalculator, _create_multilevel_index
+from dataeval.detectors.drift._nml._chunk import Chunk, Chunker
+from dataeval.detectors.drift._nml._thresholds import ConstantThreshold, Threshold
+from dataeval.outputs._base import set_metadata
+from dataeval.outputs._drift import DriftMVDCOutput
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_LGBM_HYPERPARAMS = {
+    "boosting_type": "gbdt",
+    "class_weight": None,
+    "colsample_bytree": 1.0,
+    "deterministic": True,
+    "importance_type": "split",
+    "learning_rate": 0.1,
+    "max_depth": -1,
+    "min_child_samples": 20,
+    "min_child_weight": 0.001,
+    "min_split_gain": 0.0,
+    "n_estimators": 100,
+    "num_leaves": 31,
+    "objective": None,
+    "reg_alpha": 0.0,
+    "reg_lambda": 0.0,
+    "subsample": 1.0,
+    "subsample_for_bin": 200000,
+    "subsample_freq": 0,
+    "verbosity": -1,
+}
+
+
+class DomainClassifierCalculator(AbstractCalculator):
+    """
+    DomainClassifierCalculator implementation.
+
+    Uses Drift Detection Classifier's cross validated performance as a measure of drift.
+    """
+
+    def __init__(
+        self,
+        chunker: Chunker | None = None,
+        cv_folds_num: int = 5,
+        hyperparameters: dict[str, Any] | None = None,
+        threshold: Threshold = ConstantThreshold(lower=0.45, upper=0.65),
+    ) -> None:
+        """
+        Create a new DomainClassifierCalculator instance.
+
+        Parameters
+        -----------
+        chunker : Chunker, default=None
+            The `Chunker` used to split the data sets into a lists of chunks.
+        cv_folds_num: Optional[int]
+            Number of cross-validation folds to use when calculating DC discrimination value.
+        hyperparameters : dict[str, Any], default = None
+            A dictionary used to provide your own custom hyperparameters when training the discrimination model.
+            Check out the available hyperparameter options in the
+            `LightGBM docs <https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.LGBMClassifier.html>`_.
+        threshold: Threshold, default=ConstantThreshold
+            The threshold you wish to evaluate values on. Defaults to a ConstantThreshold with lower value
+            of 0.45 and upper value of 0.65.
+        """
+        super().__init__(chunker, logger)
+
+        self.cv_folds_num = cv_folds_num
+        self.hyperparameters = DEFAULT_LGBM_HYPERPARAMS if hyperparameters is None else hyperparameters
+        self.threshold = threshold
+        self.result: DriftMVDCOutput | None = None
+
+    def _fit(self, reference_data: pd.DataFrame) -> DriftMVDCOutput:
+        """Fits the DC calculator to a set of reference data."""
+        self._x_ref = reference_data
+        result = self._calculate(data=self._x_ref)
+        result._data[("chunk", "period")] = "reference"
+
+        return result
+
+    @set_metadata
+    def _calculate(self, data: pd.DataFrame) -> DriftMVDCOutput:
+        """Calculate the data DC calculator metric for a given data set."""
+        chunks = self.chunker.split(data)
+
+        res = pd.DataFrame.from_records(
+            [
+                {
+                    **chunk.dict(),
+                    "period": "analysis",
+                    "classifier_auroc_value": self._calculate_chunk(chunk=chunk),
+                }
+                for chunk in chunks
+            ]
+        )
+
+        multilevel_index = _create_multilevel_index(chunks, "domain_classifier_auroc", ["value"])
+        res.columns = multilevel_index
+        res = res.reset_index(drop=True)
+
+        res = self._populate_alert_thresholds(res)
+
+        if self.result is None:
+            self.result = DriftMVDCOutput(results_data=res)
+        else:
+            self.result = self.result.filter(period="reference")
+            self.result._data = pd.concat([self.result._data, res], ignore_index=True)
+        return self.result
+
+    def _calculate_chunk(self, chunk: Chunk) -> float:
+        if self.result is None:
+            # Use information from chunk indices to identify reference chunk's location. This is possible because
+            # both the internal reference data copy and the chunk data were sorted by timestamp, so these
+            # indices align. This way we eliminate the need to combine these two data frames and drop duplicate rows,
+            # which is a costly operation.
+            df_X = self._x_ref
+            y = np.zeros(len(df_X), dtype=np.intp)
+            y[chunk.start_index : chunk.end_index + 1] = 1
+        else:
+            chunk_X = chunk.data
+            reference_X = self._x_ref
+            chunk_y = np.ones(len(chunk_X), dtype=np.intp)
+            reference_y = np.zeros(len(reference_X), dtype=np.intp)
+            df_X = pd.concat([reference_X, chunk_X], ignore_index=True)
+            y = np.concatenate([reference_y, chunk_y])
+
+        skf = StratifiedKFold(n_splits=self.cv_folds_num)
+        all_preds: list[NDArray[np.float32]] = []
+        all_tgts: list[NDArray[np.intp]] = []
+        for i, (train_index, test_index) in enumerate(skf.split(df_X, y)):
+            _trx = df_X.iloc[train_index]
+            _try = y[train_index]
+            _tsx = df_X.iloc[test_index]
+            _tsy = y[test_index]
+            model = LGBMClassifier(**self.hyperparameters, n_jobs=get_max_processes(), random_state=get_seed())
+            model.fit(_trx, _try)
+            preds = np.asarray(model.predict_proba(_tsx), dtype=np.float32)[:, 1]
+            all_preds.append(preds)
+            all_tgts.append(_tsy)
+
+        np_all_preds = np.concatenate(all_preds, axis=0)
+        np_all_tgts = np.concatenate(all_tgts, axis=0)
+        result = roc_auc_score(np_all_tgts, np_all_preds)
+        return 0.5 if result == np.nan else float(result)
+
+    def _populate_alert_thresholds(self, result_data: pd.DataFrame) -> pd.DataFrame:
+        if self.result is None:
+            self._threshold_values = self.threshold.calculate(
+                data=result_data.loc[:, ("domain_classifier_auroc", "value")],  # type: ignore | dataframe loc
+                lower_limit=0.0,
+                upper_limit=1.0,
+                logger=self._logger,
+            )
+
+        result_data[("domain_classifier_auroc", "upper_threshold")] = self._threshold_values[1]
+        result_data[("domain_classifier_auroc", "lower_threshold")] = self._threshold_values[0]
+        result_data[("domain_classifier_auroc", "alert")] = result_data.apply(
+            lambda row: bool(
+                row["domain_classifier_auroc", "value"] > row["domain_classifier_auroc", "upper_threshold"]
+                or row["domain_classifier_auroc", "value"] < row["domain_classifier_auroc", "lower_threshold"]
+            ),
+            axis=1,
+        )
+        return result_data