dataeval 0.85.0__py3-none-any.whl → 0.86.0__py3-none-any.whl

dataeval/__init__.py

@@ -8,7 +8,7 @@ shifts that impact performance of deployed models.
 from __future__ import annotations
 
 __all__ = ["config", "detectors", "log", "metrics", "typing", "utils", "workflows"]
-__version__ = "0.85.0"
+__version__ = "0.86.0"
 
 import logging
 

dataeval/data/_metadata.py

@@ -191,6 +191,11 @@ class Metadata:
         self._process()
         return self._image_indices
 
+    @property
+    def image_count(self) -> int:
+        self._process()
+        return int(self._image_indices.max() + 1)
+
     def _collate(self, force: bool = False):
         if self._collated and not force:
             return
@@ -359,12 +364,19 @@ class Metadata:
 
     def add_factors(self, factors: Mapping[str, ArrayLike]) -> None:
         self._merge()
-        self._processed = False
-        target_len = len(self.targets.source) if self.targets.source is not None else len(self.targets)
-        if any(len(v if isinstance(v, Sized) else as_numpy(v)) != target_len for v in factors.values()):
+
+        targets = len(self.targets.source) if self.targets.source is not None else len(self.targets)
+        images = self.image_count
+        lengths = {k: len(v if isinstance(v, Sized) else np.atleast_1d(as_numpy(v))) for k, v in factors.items()}
+        targets_match = all(f == targets for f in lengths.values())
+        images_match = targets_match if images == targets else all(f == images for f in lengths.values())
+        if not targets_match and not images_match:
             raise ValueError(
                 "The lists/arrays in the provided factors have a different length than the current metadata factors."
             )
-        merged = cast(tuple[dict[str, ArrayLike], dict[str, list[str]]], self._merged)[0]
+        merged = cast(dict[str, ArrayLike], self._merged[0] if self._merged is not None else {})
         for k, v in factors.items():
-            merged[k] = v
+            v = as_numpy(v)
+            merged[k] = v if (self.targets.source is None or lengths[k] == targets) else v[self.targets.source]
+
+        self._processed = False

dataeval/data/_selection.py

@@ -120,7 +120,7 @@ class Select(AnnotatedDataset[_TDatum]):
 
     def _apply_subselection(self, datum: _TDatum, index: int) -> _TDatum:
         for subselection, indices in self._subselections:
-            datum = subselection(datum) if index in indices else datum
+            datum = subselection(datum) if self._selection[index] in indices else datum
         return datum
 
     def __getitem__(self, index: int) -> _TDatum:

dataeval/data/selections/_classfilter.py

@@ -10,7 +10,6 @@ from numpy.typing import NDArray
 from dataeval.data._selection import Select, Selection, SelectionStage, Subselection
 from dataeval.typing import Array, ObjectDetectionDatum, ObjectDetectionTarget, SegmentationDatum, SegmentationTarget
 from dataeval.utils._array import as_numpy
-from dataeval.utils.data.metadata import flatten
 
 
 class ClassFilter(Selection[Any]):
@@ -96,13 +95,15 @@ class ClassFilterSubSelection(Subselection[Any]):
     def __init__(self, classes: Sequence[int]) -> None:
         self.classes = classes
 
+    def _filter(self, d: dict[str, Any], mask: NDArray[np.bool_]) -> dict[str, Any]:
+        return {k: self._filter(v, mask) if isinstance(v, dict) else _try_mask_object(v, mask) for k, v in d.items()}
+
     def __call__(self, datum: _TDatum) -> _TDatum:
         # build a mask for any arrays
         image, target, metadata = datum
 
         mask = np.isin(as_numpy(target.labels), self.classes)
-        flattened_metadata = flatten(metadata)[0]
-        filtered_metadata = {k: _try_mask_object(v, mask) for k, v in flattened_metadata.items()}
+        filtered_metadata = self._filter(metadata, mask)
 
         # return a masked datum
         filtered_datum = image, ClassFilterTarget(target, mask), filtered_metadata

dataeval/detectors/drift/__init__.py

@@ -7,6 +7,8 @@ __all__ = [
     "DriftKS",
     "DriftMMD",
     "DriftMMDOutput",
+    "DriftMVDC",
+    "DriftMVDCOutput",
     "DriftOutput",
     "DriftUncertainty",
     "UpdateStrategy",
@@ -18,5 +20,6 @@ from dataeval.detectors.drift._base import UpdateStrategy
 from dataeval.detectors.drift._cvm import DriftCVM
 from dataeval.detectors.drift._ks import DriftKS
 from dataeval.detectors.drift._mmd import DriftMMD
+from dataeval.detectors.drift._mvdc import DriftMVDC
 from dataeval.detectors.drift._uncertainty import DriftUncertainty
-from dataeval.outputs._drift import DriftMMDOutput, DriftOutput
+from dataeval.outputs._drift import DriftMMDOutput, DriftMVDCOutput, DriftOutput

dataeval/detectors/drift/_mvdc.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+import pandas as pd
+from numpy.typing import ArrayLike
+
+if TYPE_CHECKING:
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+from dataeval.detectors.drift._nml._chunk import CountBasedChunker, SizeBasedChunker
+from dataeval.detectors.drift._nml._domainclassifier import DomainClassifierCalculator
+from dataeval.detectors.drift._nml._thresholds import ConstantThreshold
+from dataeval.outputs._drift import DriftMVDCOutput
+from dataeval.utils._array import flatten
+
+
+class DriftMVDC:
+    """Multivariant Domain Classifier
+
+    Parameters
+    ----------
+    n_folds : int, default 5
+        Number of cross-validation (CV) folds.
+    chunk_size : int or None, default None
+        Number of samples in a chunk used in CV, will get one metric & prediction per chunk.
+    chunk_count : int or None, default None
+        Number of total chunks used in CV, will get one metric & prediction per chunk.
+    threshold : Tuple[float, float], default (0.45, 0.65)
+        (lower, upper) metric bounds on roc_auc for identifying :term:`drift<Drift>`.
+    """
+
+    def __init__(
+        self,
+        n_folds: int = 5,
+        chunk_size: int | None = None,
+        chunk_count: int | None = None,
+        threshold: tuple[float, float] = (0.45, 0.65),
+    ) -> None:
+        self.threshold: tuple[float, float] = max(0.0, min(threshold)), min(1.0, max(threshold))
+        chunker = (
+            CountBasedChunker(10 if chunk_count is None else chunk_count)
+            if chunk_size is None
+            else SizeBasedChunker(chunk_size)
+        )
+        self._calc = DomainClassifierCalculator(
+            cv_folds_num=n_folds,
+            chunker=chunker,
+            threshold=ConstantThreshold(lower=self.threshold[0], upper=self.threshold[1]),
+        )
+
+    def fit(self, x_ref: ArrayLike) -> Self:
+        """
+        Fit the domain classifier on the training dataframe
+
+        Parameters
+        ----------
+        x_ref : ArrayLike
+            Reference data with dim[n_samples, n_features].
+
+        Returns
+        -------
+        Self
+
+        """
+        # for 1D input, assume that is 1 sample: dim[1,n_features]
+        self.x_ref: pd.DataFrame = pd.DataFrame(flatten(np.atleast_2d(np.asarray(x_ref))))
+        self.n_features: int = self.x_ref.shape[-1]
+        self._calc.fit(self.x_ref)
+        return self
+
+    def predict(self, x: ArrayLike) -> DriftMVDCOutput:
+        """
+        Perform :term:`inference<Inference>` on the test dataframe
+
+        Parameters
+        ----------
+        x : ArrayLike
+            Test (analysis) data with dim[n_samples, n_features].
+
+        Returns
+        -------
+        DomainClassifierDriftResult
+        """
+        self.x_test: pd.DataFrame = pd.DataFrame(flatten(np.atleast_2d(np.asarray(x))))
+        if self.x_test.shape[-1] != self.n_features:
+            raise ValueError("Reference and test embeddings have different number of features")
+
+        return self._calc.calculate(self.x_test)

dataeval/detectors/drift/_nml/__init__.py

@@ -0,0 +1,6 @@
+"""
+Source code derived from NannyML 0.13.0
+https://github.com/NannyML/nannyml/
+
+Licensed under Apache Software License (Apache 2.0)
+"""

dataeval/detectors/drift/_nml/_base.py

@@ -0,0 +1,68 @@
+"""
+Source code derived from NannyML 0.13.0
+https://github.com/NannyML/nannyml/blob/main/nannyml/base.py
+
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from logging import Logger
+from typing import Sequence
+
+import pandas as pd
+from typing_extensions import Self
+
+from dataeval.detectors.drift._nml._chunk import Chunk, Chunker, CountBasedChunker
+from dataeval.outputs._drift import DriftMVDCOutput
+
+
+def _validate(data: pd.DataFrame, expected_features: int | None = None) -> int:
+    if data.empty:
+        raise ValueError("data contains no rows. Please provide a valid data set.")
+    if expected_features is not None and data.shape[-1] != expected_features:
+        raise ValueError(f"expected '{expected_features}' features in data set:\n\t{data}")
+    return data.shape[-1]
+
+
+def _create_multilevel_index(chunks: Sequence[Chunk], result_group_name: str, result_column_names: Sequence[str]):
+    chunk_column_names = (*chunks[0].KEYS, "period")
+    chunk_tuples = [("chunk", chunk_column_name) for chunk_column_name in chunk_column_names]
+    result_tuples = [(result_group_name, column_name) for column_name in result_column_names]
+    return pd.MultiIndex.from_tuples(chunk_tuples + result_tuples)
+
+
+class AbstractCalculator(ABC):
+    """Base class for drift calculation."""
+
+    def __init__(self, chunker: Chunker | None = None, logger: Logger | None = None):
+        self.chunker = chunker if isinstance(chunker, Chunker) else CountBasedChunker(10)
+        self.result: DriftMVDCOutput | None = None
+        self.n_features: int | None = None
+        self._logger = logger if isinstance(logger, Logger) else logging.getLogger(__name__)
+
+    def fit(self, reference_data: pd.DataFrame) -> Self:
+        """Trains the calculator using reference data."""
+        self.n_features = _validate(reference_data)
+
+        self._logger.debug(f"fitting {str(self)}")
+        self.result = self._fit(reference_data)
+        return self
+
+    def calculate(self, data: pd.DataFrame) -> DriftMVDCOutput:
+        """Performs a calculation on the provided data."""
+        if self.result is None:
+            raise RuntimeError("must run fit with reference data before running calculate")
+        _validate(data, self.n_features)
+
+        self._logger.debug(f"calculating {str(self)}")
+        self.result = self._calculate(data)
+        return self.result
+
+    @abstractmethod
+    def _fit(self, reference_data: pd.DataFrame) -> DriftMVDCOutput: ...
+
+    @abstractmethod
+    def _calculate(self, data: pd.DataFrame) -> DriftMVDCOutput: ...

dataeval/detectors/drift/_nml/_chunk.py

@@ -0,0 +1,404 @@
+"""
+NannyML module providing intelligent splitting of data into chunks.
+
+Source code derived from NannyML 0.13.0
+https://github.com/NannyML/nannyml/blob/main/nannyml/chunk.py
+
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from __future__ import annotations
+
+import copy
+import logging
+import warnings
+from abc import ABC, abstractmethod
+from typing import Any, Generic, Literal, Sequence, TypeVar, cast
+
+import pandas as pd
+from dateutil.parser import ParserError
+from pandas import Index, Period
+from typing_extensions import Self
+
+logger = logging.getLogger(__name__)
+
+
+class Chunk(ABC):
+    """A subset of data that acts as a logical unit during calculations."""
+
+    KEYS: Sequence[str]
+
+    def __init__(
+        self,
+        data: pd.DataFrame,
+    ):
+        self.key: str
+        self.data = data
+
+        self.start_index: int = -1
+        self.end_index: int = -1
+        self.chunk_index: int = -1
+
+    def __repr__(self):
+        attr_str = ", ".join([f"{k}={v}" for k, v in self.dict().items()])
+        return f"{self.__class__.__name__}(data=pd.DataFrame(shape={self.data.shape}), {attr_str})"
+
+    def __len__(self):
+        return self.data.shape[0]
+
+    @abstractmethod
+    def __add__(self, other: Self) -> Self: ...
+
+    @abstractmethod
+    def __lt__(self, other: Self) -> bool: ...
+
+    @abstractmethod
+    def dict(self) -> dict[str, Any]: ...
+
+
+class IndexChunk(Chunk):
+    """Creates a new chunk.
+
+    Parameters
+    ----------
+    data : DataFrame, required
+        The data to be contained within the chunk.
+    start_datetime: datetime
+        The starting point in time for this chunk.
+    end_datetime: datetime
+        The end point in time for this chunk.
+    """
+
+    KEYS = ("key", "chunk_index", "start_index", "end_index")
+
+    def __init__(
+        self,
+        data: pd.DataFrame,
+        start_index: int,
+        end_index: int,
+    ):
+        super().__init__(data)
+        self.key = f"[{start_index}:{end_index}]"
+        self.start_index: int = start_index
+        self.end_index: int = end_index
+
+    def __lt__(self, other: Self) -> bool:
+        return self.end_index < other.start_index
+
+    def __add__(self, other: Self) -> Self:
+        a, b = (self, other) if self < other else (other, self)
+        result = copy.deepcopy(a)
+        result.data = pd.concat([a.data, b.data])
+        result.end_index = b.end_index
+        return result
+
+    def dict(self) -> dict[str, Any]:
+        return dict(zip(self.KEYS, (self.key, self.chunk_index, self.start_index, self.end_index)))
+
+
+class PeriodChunk(Chunk):
+    """Creates a new chunk.
+
+    Parameters
+    ----------
+    data : DataFrame, required
+        The data to be contained within the chunk.
+    start_datetime: datetime
+        The starting point in time for this chunk.
+    end_datetime: datetime
+        The end point in time for this chunk.
+    chunk_size : int
+        The size of the chunk.
+    """
+
+    KEYS = ("key", "chunk_index", "start_date", "end_date", "chunk_size")
+
+    def __init__(self, data: pd.DataFrame, period: Period, chunk_size: int):
+        super().__init__(data)
+        self.key = str(period)
+        self.start_datetime = period.start_time
+        self.end_datetime = period.end_time
+        self.chunk_size = chunk_size
+
+    def __lt__(self, other: Self) -> bool:
+        return self.end_datetime < other.start_datetime
+
+    def __add__(self, other: Self) -> Self:
+        a, b = (self, other) if self < other else (other, self)
+        result = copy.deepcopy(a)
+        result.data = pd.concat([a.data, b.data])
+        result.end_datetime = b.end_datetime
+        result.chunk_size += b.chunk_size
+        return result
+
+    def dict(self) -> dict[str, Any]:
+        return dict(
+            zip(self.KEYS, (self.key, self.chunk_index, self.start_datetime, self.end_datetime, self.chunk_size))
+        )
+
+
+TChunk = TypeVar("TChunk", bound=Chunk)
+
+
+class Chunker(Generic[TChunk]):
+    """Base class for Chunker implementations.
+
+    Inheriting classes will split a DataFrame into a list of Chunks.
+    They will do this based on several constraints, e.g. observation timestamps, number of observations per Chunk
+    or a preferred number of Chunks.
+    """
+
+    def split(self, data: pd.DataFrame) -> list[TChunk]:
+        """Splits a given data frame into a list of chunks.
+
+        This method provides a uniform interface across Chunker implementations to keep them interchangeable.
+
+        After performing the implementation-specific `_split` method, there are some checks on the resulting chunk list.
+
+        If the total number of chunks is low a warning will be written out to the logs.
+
+        We dynamically determine the optimal minimum number of observations per chunk and then check if the resulting
+        chunks contain at least as many. If there are any underpopulated chunks a warning will be written out in
+        the logs.
+
+        Parameters
+        ----------
+        data: DataFrame
+            The data to be split into chunks
+
+        Returns
+        -------
+        chunks: List[Chunk]
+            The list of chunks
+
+        """
+        if data.shape[0] == 0:
+            return []
+
+        chunks = self._split(data)
+        for chunk_index, chunk in enumerate(chunks):
+            chunk.start_index = cast(int, chunk.data.index.min())
+            chunk.end_index = cast(int, chunk.data.index.max())
+            chunk.chunk_index = chunk_index
+
+        if len(chunks) < 6:
+            # TODO wording
+            warnings.warn(
+                "The resulting number of chunks is too low. "
+                "Please consider splitting your data in a different way or continue at your own risk."
+            )
+
+        return chunks
+
+    @abstractmethod
+    def _split(self, data: pd.DataFrame) -> list[TChunk]: ...
+
+
+class PeriodBasedChunker(Chunker[PeriodChunk]):
+    """A Chunker that will split data into Chunks based on a date column in the data.
+
+    Examples
+    --------
+    Chunk using monthly periods and providing a column name
+
+    >>> from nannyml.chunk import PeriodBasedChunker
+    >>> df = pd.read_parquet("/path/to/my/data.pq")
+    >>> chunker = PeriodBasedChunker(timestamp_column_name="observation_date", offset="M")
+    >>> chunks = chunker.split(data=df)
+
+    Or chunk using weekly periods
+
+    >>> from nannyml.chunk import PeriodBasedChunker
+    >>> df = pd.read_parquet("/path/to/my/data.pq")
+    >>> chunker = PeriodBasedChunker(timestamp_column_name=df["observation_date"], offset="W", minimum_chunk_size=50)
+    >>> chunks = chunker.split(data=df)
+
+    """
+
+    def __init__(self, timestamp_column_name: str, offset: str = "W") -> None:
+        """Creates a new PeriodBasedChunker.
+
+        Parameters
+        ----------
+        timestamp_column_name : str
+            The column name containing the timestamp to chunk on
+        offset : str
+            A frequency string representing a pandas.tseries.offsets.DateOffset.
+            The offset determines how the time-based grouping will occur. A list of possible values
+            can be found at <https://pandas.pydata.org/docs/user_guide/timeseries.html#offset-aliases>.
+        """
+        self.timestamp_column_name = timestamp_column_name
+        self.offset = offset
+
+    def _split(self, data: pd.DataFrame) -> list[PeriodChunk]:
+        chunks = []
+        if self.timestamp_column_name is None:
+            raise ValueError("timestamp_column_name must be provided")
+        if self.timestamp_column_name not in data:
+            raise ValueError(f"timestamp column '{self.timestamp_column_name}' not in columns")
+
+        try:
+            grouped = data.groupby(pd.to_datetime(data[self.timestamp_column_name]).dt.to_period(self.offset))
+        except ParserError:
+            raise ValueError(
+                f"could not parse date_column '{self.timestamp_column_name}' values as dates."
+                f"Please verify if you've specified the correct date column."
+            )
+
+        for k, v in grouped.groups.items():
+            period, index = cast(Period, k), cast(Index, v)
+            chunk = PeriodChunk(
+                data=grouped.get_group(period),  # type: ignore | dataframe
+                period=period,
+                chunk_size=len(index),
+            )
+            chunks.append(chunk)
+
+        return chunks
+
+
+class SizeBasedChunker(Chunker[IndexChunk]):
+    """A Chunker that will split data into Chunks based on the preferred number of observations per Chunk.
+
+    Notes
+    -----
+    - Chunks are adjacent, not overlapping
+    - There may be "incomplete" chunks, as the remainder of observations after dividing by `chunk_size`
+      will form a chunk of their own.
+
+    Examples
+    --------
+    Chunk using monthly periods and providing a column name
+
+    >>> from nannyml.chunk import SizeBasedChunker
+    >>> df = pd.read_parquet("/path/to/my/data.pq")
+    >>> chunker = SizeBasedChunker(chunk_size=2000, incomplete="drop")
+    >>> chunks = chunker.split(data=df)
+
+    """
+
+    def __init__(
+        self,
+        chunk_size: int,
+        incomplete: Literal["append", "drop", "keep"] = "keep",
+    ):
+        """Create a new SizeBasedChunker.
+
+        Parameters
+        ----------
+        chunk_size: int
+            The preferred size of the resulting Chunks, i.e. the number of observations in each Chunk.
+        incomplete: str, default='keep'
+            Choose how to handle any leftover observations that don't make up a full Chunk.
+            The following options are available:
+
+            - ``'drop'``: drop the leftover observations
+            - ``'keep'``: keep the incomplete Chunk (containing less than ``chunk_size`` observations)
+            - ``'append'``: append leftover observations to the last complete Chunk (overfilling it)
+
+            Defaults to ``'keep'``.
+
+        Returns
+        -------
+        chunker: a size-based instance used to split data into Chunks of a constant size.
+
+        """
+        if not isinstance(chunk_size, int) or chunk_size <= 0:
+            raise ValueError(f"chunk_size={chunk_size} is invalid - provide an integer greater than 0")
+        if incomplete not in ("append", "drop", "keep"):
+            raise ValueError(f"incomplete={incomplete} is invalid - must be one of ['append', 'drop', 'keep']")
+
+        self.chunk_size = chunk_size
+        self.incomplete = incomplete
+
+    def _split(self, data: pd.DataFrame) -> list[IndexChunk]:
+        def _create_chunk(index: int, data: pd.DataFrame, chunk_size: int) -> IndexChunk:
+            chunk_data = data.iloc[index : index + chunk_size]
+            chunk = IndexChunk(
+                data=chunk_data,
+                start_index=index,
+                end_index=index + chunk_size - 1,
+            )
+            return chunk
+
+        chunks = [
+            _create_chunk(index=i, data=data, chunk_size=self.chunk_size)
+            for i in range(0, data.shape[0], self.chunk_size)
+            if i + self.chunk_size - 1 < len(data)
+        ]
+
+        # deal with unassigned observations
+        if data.shape[0] % self.chunk_size != 0 and self.incomplete != "drop":
+            incomplete_chunk = _create_chunk(
+                index=self.chunk_size * (data.shape[0] // self.chunk_size),
+                data=data,
+                chunk_size=(data.shape[0] % self.chunk_size),
+            )
+            if self.incomplete == "append":
+                chunks[-1] += incomplete_chunk
+            else:
+                chunks += [incomplete_chunk]
+
+        return chunks
+
+
+class CountBasedChunker(Chunker[IndexChunk]):
+    """A Chunker that will split data into chunks based on the preferred number of total chunks.
+
+    Notes
+    -----
+    - Chunks are adjacent, not overlapping
+    - There may be "incomplete" chunks, as the remainder of observations after dividing by `chunk_size`
+      will form a chunk of their own.
+
+    Examples
+    --------
+    >>> from nannyml.chunk import CountBasedChunker
+    >>> df = pd.read_parquet("/path/to/my/data.pq")
+    >>> chunker = CountBasedChunker(chunk_number=100)
+    >>> chunks = chunker.split(data=df)
+
+    """
+
+    def __init__(
+        self,
+        chunk_number: int,
+        incomplete: Literal["append", "drop", "keep"] = "keep",
+    ):
+        """Creates a new CountBasedChunker.
+
+        It will calculate the amount of observations per chunk based on the given chunk count.
+        It then continues to split the data into chunks just like a SizeBasedChunker does.
+
+        Parameters
+        ----------
+        chunk_number: int
+            The amount of chunks to split the data in.
+        incomplete: str, default='keep'
+            Choose how to handle any leftover observations that don't make up a full Chunk.
+            The following options are available:
+
+            - ``'drop'``: drop the leftover observations
+            - ``'keep'``: keep the incomplete Chunk (containing less than ``chunk_size`` observations)
+            - ``'append'``: append leftover observations to the last complete Chunk (overfilling it)
+
+            Defaults to ``'keep'``.
+
+        Returns
+        -------
+        chunker: CountBasedChunker
+
+        """
+        if not isinstance(chunk_number, int) or chunk_number <= 0:
+            raise ValueError(f"given chunk_number {chunk_number} is invalid - provide an integer greater than 0")
+        if incomplete not in ("append", "drop", "keep"):
+            raise ValueError(f"incomplete={incomplete} is invalid - must be one of ['append', 'drop', 'keep']")
+
+        self.chunk_number = chunk_number
+        self.incomplete: Literal["append", "drop", "keep"] = incomplete
+
+    def _split(self, data: pd.DataFrame) -> list[IndexChunk]:
+        chunk_size = data.shape[0] // self.chunk_number
+        chunker = SizeBasedChunker(chunk_size, self.incomplete)
+        chunks = chunker.split(data=data)
+        return chunks