dataeval 0.85.0__tar.gz → 0.86.0__tar.gz

{dataeval-0.85.0 → dataeval-0.86.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataeval
-Version: 0.85.0
+Version: 0.86.0
 Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
 Home-page: https://dataeval.ai/
 License: MIT
@@ -23,10 +23,11 @@ Classifier: Topic :: Scientific/Engineering
 Provides-Extra: all
 Requires-Dist: defusedxml (>=0.7.1)
 Requires-Dist: fast_hdbscan (==0.2.0)
+Requires-Dist: lightgbm (>=4)
 Requires-Dist: matplotlib (>=3.7.1) ; extra == "all"
 Requires-Dist: numba (>=0.59.1)
 Requires-Dist: numpy (>=1.24.2)
-Requires-Dist: pandas (>=2.0) ; extra == "all"
+Requires-Dist: pandas (>=2.0)
 Requires-Dist: pillow (>=10.3.0)
 Requires-Dist: requests
 Requires-Dist: scikit-learn (>=1.5.0)

{dataeval-0.85.0 → dataeval-0.86.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dataeval"
-version = "0.85.0" # dynamic
+version = "0.86.0" # dynamic
 description = "DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks"
 license = "MIT"
 readme = "README.md"
@@ -44,8 +44,10 @@ packages = [
 python = ">=3.9,<3.13"
 defusedxml = {version = ">=0.7.1"}
 fast_hdbscan = {version = "0.2.0"}  # 0.2.1 hits a bug in condense_tree comparing float to none
+lightgbm = {version = ">=4"}
 numba = {version = ">=0.59.1"}
 numpy = {version = ">=1.24.2"}
+pandas = {version = ">=2.0"}
 pillow = {version = ">=10.3.0"}
 requests = {version = "*"}
 scipy = {version = ">=1.10"}
@@ -58,10 +60,9 @@ xxhash = {version = ">=3.3"}
 
 # optional
 matplotlib = {version = ">=3.7.1", optional = true}
-pandas = {version = ">=2.0", optional = true}
 
 [tool.poetry.extras]
-all = ["matplotlib", "pandas"]
+all = ["matplotlib"]
 
 [tool.poetry.group.dev]
 optional = true
@@ -132,6 +133,7 @@ markers = [
   "required: marks tests for required features",
   "optional: marks tests for optional features",
   "requires_all: marks tests that require the all extras",
+  "cuda: marks tests that require cuda",
 ]
 
 [tool.coverage.run]

{dataeval-0.85.0 → dataeval-0.86.0}/src/dataeval/__init__.py

@@ -8,7 +8,7 @@ shifts that impact performance of deployed models.
 from __future__ import annotations
 
 __all__ = ["config", "detectors", "log", "metrics", "typing", "utils", "workflows"]
-__version__ = "0.85.0"
+__version__ = "0.86.0"
 
 import logging
 

{dataeval-0.85.0 → dataeval-0.86.0}/src/dataeval/data/_metadata.py

@@ -191,6 +191,11 @@ class Metadata:
         self._process()
         return self._image_indices
 
+    @property
+    def image_count(self) -> int:
+        self._process()
+        return int(self._image_indices.max() + 1)
+
     def _collate(self, force: bool = False):
         if self._collated and not force:
             return
@@ -359,12 +364,19 @@ class Metadata:
 
     def add_factors(self, factors: Mapping[str, ArrayLike]) -> None:
         self._merge()
-        self._processed = False
-        target_len = len(self.targets.source) if self.targets.source is not None else len(self.targets)
-        if any(len(v if isinstance(v, Sized) else as_numpy(v)) != target_len for v in factors.values()):
+
+        targets = len(self.targets.source) if self.targets.source is not None else len(self.targets)
+        images = self.image_count
+        lengths = {k: len(v if isinstance(v, Sized) else np.atleast_1d(as_numpy(v))) for k, v in factors.items()}
+        targets_match = all(f == targets for f in lengths.values())
+        images_match = targets_match if images == targets else all(f == images for f in lengths.values())
+        if not targets_match and not images_match:
             raise ValueError(
                 "The lists/arrays in the provided factors have a different length than the current metadata factors."
             )
-        merged = cast(tuple[dict[str, ArrayLike], dict[str, list[str]]], self._merged)[0]
+        merged = cast(dict[str, ArrayLike], self._merged[0] if self._merged is not None else {})
         for k, v in factors.items():
-            merged[k] = v
+            v = as_numpy(v)
+            merged[k] = v if (self.targets.source is None or lengths[k] == targets) else v[self.targets.source]
+
+        self._processed = False

{dataeval-0.85.0 → dataeval-0.86.0}/src/dataeval/data/_selection.py

@@ -120,7 +120,7 @@ class Select(AnnotatedDataset[_TDatum]):
 
     def _apply_subselection(self, datum: _TDatum, index: int) -> _TDatum:
         for subselection, indices in self._subselections:
-            datum = subselection(datum) if index in indices else datum
+            datum = subselection(datum) if self._selection[index] in indices else datum
         return datum
 
     def __getitem__(self, index: int) -> _TDatum:

{dataeval-0.85.0 → dataeval-0.86.0}/src/dataeval/data/selections/_classfilter.py

@@ -10,7 +10,6 @@ from numpy.typing import NDArray
 from dataeval.data._selection import Select, Selection, SelectionStage, Subselection
 from dataeval.typing import Array, ObjectDetectionDatum, ObjectDetectionTarget, SegmentationDatum, SegmentationTarget
 from dataeval.utils._array import as_numpy
-from dataeval.utils.data.metadata import flatten
 
 
 class ClassFilter(Selection[Any]):
@@ -96,13 +95,15 @@ class ClassFilterSubSelection(Subselection[Any]):
     def __init__(self, classes: Sequence[int]) -> None:
         self.classes = classes
 
+    def _filter(self, d: dict[str, Any], mask: NDArray[np.bool_]) -> dict[str, Any]:
+        return {k: self._filter(v, mask) if isinstance(v, dict) else _try_mask_object(v, mask) for k, v in d.items()}
+
     def __call__(self, datum: _TDatum) -> _TDatum:
         # build a mask for any arrays
         image, target, metadata = datum
 
         mask = np.isin(as_numpy(target.labels), self.classes)
-        flattened_metadata = flatten(metadata)[0]
-        filtered_metadata = {k: _try_mask_object(v, mask) for k, v in flattened_metadata.items()}
+        filtered_metadata = self._filter(metadata, mask)
 
         # return a masked datum
         filtered_datum = image, ClassFilterTarget(target, mask), filtered_metadata

{dataeval-0.85.0 → dataeval-0.86.0}/src/dataeval/detectors/drift/__init__.py

@@ -7,6 +7,8 @@ __all__ = [
     "DriftKS",
     "DriftMMD",
     "DriftMMDOutput",
+    "DriftMVDC",
+    "DriftMVDCOutput",
     "DriftOutput",
     "DriftUncertainty",
     "UpdateStrategy",
@@ -18,5 +20,6 @@ from dataeval.detectors.drift._base import UpdateStrategy
 from dataeval.detectors.drift._cvm import DriftCVM
 from dataeval.detectors.drift._ks import DriftKS
 from dataeval.detectors.drift._mmd import DriftMMD
+from dataeval.detectors.drift._mvdc import DriftMVDC
 from dataeval.detectors.drift._uncertainty import DriftUncertainty
-from dataeval.outputs._drift import DriftMMDOutput, DriftOutput
+from dataeval.outputs._drift import DriftMMDOutput, DriftMVDCOutput, DriftOutput

dataeval-0.86.0/src/dataeval/detectors/drift/_mvdc.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+import pandas as pd
+from numpy.typing import ArrayLike
+
+if TYPE_CHECKING:
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+from dataeval.detectors.drift._nml._chunk import CountBasedChunker, SizeBasedChunker
+from dataeval.detectors.drift._nml._domainclassifier import DomainClassifierCalculator
+from dataeval.detectors.drift._nml._thresholds import ConstantThreshold
+from dataeval.outputs._drift import DriftMVDCOutput
+from dataeval.utils._array import flatten
+
+
+class DriftMVDC:
+    """Multivariant Domain Classifier
+
+    Parameters
+    ----------
+    n_folds : int, default 5
+        Number of cross-validation (CV) folds.
+    chunk_size : int or None, default None
+        Number of samples in a chunk used in CV, will get one metric & prediction per chunk.
+    chunk_count : int or None, default None
+        Number of total chunks used in CV, will get one metric & prediction per chunk.
+    threshold : Tuple[float, float], default (0.45, 0.65)
+        (lower, upper) metric bounds on roc_auc for identifying :term:`drift<Drift>`.
+    """
+
+    def __init__(
+        self,
+        n_folds: int = 5,
+        chunk_size: int | None = None,
+        chunk_count: int | None = None,
+        threshold: tuple[float, float] = (0.45, 0.65),
+    ) -> None:
+        self.threshold: tuple[float, float] = max(0.0, min(threshold)), min(1.0, max(threshold))
+        chunker = (
+            CountBasedChunker(10 if chunk_count is None else chunk_count)
+            if chunk_size is None
+            else SizeBasedChunker(chunk_size)
+        )
+        self._calc = DomainClassifierCalculator(
+            cv_folds_num=n_folds,
+            chunker=chunker,
+            threshold=ConstantThreshold(lower=self.threshold[0], upper=self.threshold[1]),
+        )
+
+    def fit(self, x_ref: ArrayLike) -> Self:
+        """
+        Fit the domain classifier on the training dataframe
+
+        Parameters
+        ----------
+        x_ref : ArrayLike
+            Reference data with dim[n_samples, n_features].
+
+        Returns
+        -------
+        Self
+
+        """
+        # for 1D input, assume that is 1 sample: dim[1,n_features]
+        self.x_ref: pd.DataFrame = pd.DataFrame(flatten(np.atleast_2d(np.asarray(x_ref))))
+        self.n_features: int = self.x_ref.shape[-1]
+        self._calc.fit(self.x_ref)
+        return self
+
+    def predict(self, x: ArrayLike) -> DriftMVDCOutput:
+        """
+        Perform :term:`inference<Inference>` on the test dataframe
+
+        Parameters
+        ----------
+        x : ArrayLike
+            Test (analysis) data with dim[n_samples, n_features].
+
+        Returns
+        -------
+        DomainClassifierDriftResult
+        """
+        self.x_test: pd.DataFrame = pd.DataFrame(flatten(np.atleast_2d(np.asarray(x))))
+        if self.x_test.shape[-1] != self.n_features:
+            raise ValueError("Reference and test embeddings have different number of features")
+
+        return self._calc.calculate(self.x_test)

dataeval-0.86.0/src/dataeval/detectors/drift/_nml/__init__.py

@@ -0,0 +1,6 @@
+"""
+Source code derived from NannyML 0.13.0
+https://github.com/NannyML/nannyml/
+
+Licensed under Apache Software License (Apache 2.0)
+"""

dataeval-0.86.0/src/dataeval/detectors/drift/_nml/_base.py

@@ -0,0 +1,68 @@
+"""
+Source code derived from NannyML 0.13.0
+https://github.com/NannyML/nannyml/blob/main/nannyml/base.py
+
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from logging import Logger
+from typing import Sequence
+
+import pandas as pd
+from typing_extensions import Self
+
+from dataeval.detectors.drift._nml._chunk import Chunk, Chunker, CountBasedChunker
+from dataeval.outputs._drift import DriftMVDCOutput
+
+
+def _validate(data: pd.DataFrame, expected_features: int | None = None) -> int:
+    if data.empty:
+        raise ValueError("data contains no rows. Please provide a valid data set.")
+    if expected_features is not None and data.shape[-1] != expected_features:
+        raise ValueError(f"expected '{expected_features}' features in data set:\n\t{data}")
+    return data.shape[-1]
+
+
+def _create_multilevel_index(chunks: Sequence[Chunk], result_group_name: str, result_column_names: Sequence[str]):
+    chunk_column_names = (*chunks[0].KEYS, "period")
+    chunk_tuples = [("chunk", chunk_column_name) for chunk_column_name in chunk_column_names]
+    result_tuples = [(result_group_name, column_name) for column_name in result_column_names]
+    return pd.MultiIndex.from_tuples(chunk_tuples + result_tuples)
+
+
+class AbstractCalculator(ABC):
+    """Base class for drift calculation."""
+
+    def __init__(self, chunker: Chunker | None = None, logger: Logger | None = None):
+        self.chunker = chunker if isinstance(chunker, Chunker) else CountBasedChunker(10)
+        self.result: DriftMVDCOutput | None = None
+        self.n_features: int | None = None
+        self._logger = logger if isinstance(logger, Logger) else logging.getLogger(__name__)
+
+    def fit(self, reference_data: pd.DataFrame) -> Self:
+        """Trains the calculator using reference data."""
+        self.n_features = _validate(reference_data)
+
+        self._logger.debug(f"fitting {str(self)}")
+        self.result = self._fit(reference_data)
+        return self
+
+    def calculate(self, data: pd.DataFrame) -> DriftMVDCOutput:
+        """Performs a calculation on the provided data."""
+        if self.result is None:
+            raise RuntimeError("must run fit with reference data before running calculate")
+        _validate(data, self.n_features)
+
+        self._logger.debug(f"calculating {str(self)}")
+        self.result = self._calculate(data)
+        return self.result
+
+    @abstractmethod
+    def _fit(self, reference_data: pd.DataFrame) -> DriftMVDCOutput: ...
+
+    @abstractmethod
+    def _calculate(self, data: pd.DataFrame) -> DriftMVDCOutput: ...