dataeval 0.64.0__py3-none-any.whl → 0.66.0__py3-none-any.whl

dataeval/_internal/detectors/drift/base.py

@@ -6,14 +6,64 @@ Original code Copyright (c) 2023 Seldon Technologies Ltd
 Licensed under Apache Software License (Apache 2.0)
 """
 
+from __future__ import annotations
+
 from abc import ABC, abstractmethod
+from dataclasses import dataclass
 from functools import wraps
-from typing import Callable, Dict, Literal, Optional, Tuple, Union
+from typing import Callable, Literal
 
 import numpy as np
-from numpy.typing import ArrayLike
+from numpy.typing import ArrayLike, NDArray
 
 from dataeval._internal.interop import to_numpy
+from dataeval._internal.output import OutputMetadata, set_metadata
+
+
+@dataclass(frozen=True)
+class DriftBaseOutput(OutputMetadata):
+    """
+    Output class for Drift
+
+    Attributes
+    ----------
+    is_drift : bool
+        Drift prediction for the images
+    threshold : float
+        Threshold after multivariate correction if needed
+    """
+
+    is_drift: bool
+    threshold: float
+
+
+@dataclass(frozen=True)
+class DriftOutput(DriftBaseOutput):
+    """
+    Output class for DriftCVM and DriftKS
+
+    Attributes
+    ----------
+    is_drift : bool
+        Drift prediction for the images
+    threshold : float
+        Threshold after multivariate correction if needed
+    feature_drift : NDArray
+        Feature-level array of images detected to have drifted
+    feature_threshold : float
+        Feature-level threshold to determine drift
+    p_vals : NDArray
+        Feature-level p-values
+    distances : NDArray
+        Feature-level distances
+    """
+
+    # is_drift: bool
+    # threshold: float
+    feature_drift: NDArray[np.bool_]
+    feature_threshold: float
+    p_vals: NDArray[np.float32]
+    distances: NDArray[np.float32]
 
 
 def update_x_ref(fn):
@@ -48,11 +98,20 @@ def preprocess_x(fn):
 
 
 class UpdateStrategy(ABC):
+    """
+    Updates reference dataset for drift detector
+
+    Parameters
+    ----------
+    n : int
+        Update with last n instances seen by the detector.
+    """
+
     def __init__(self, n: int):
         self.n = n
 
     @abstractmethod
-    def __call__(self, x_ref: np.ndarray, x: np.ndarray, count: int) -> np.ndarray:
+    def __call__(self, x_ref: NDArray, x: NDArray, count: int) -> NDArray:
         """Abstract implementation of update strategy"""
 
 
@@ -66,7 +125,7 @@ class LastSeenUpdate(UpdateStrategy):
         Update with last n instances seen by the detector.
     """
 
-    def __call__(self, x_ref: np.ndarray, x: np.ndarray, count: int) -> np.ndarray:
+    def __call__(self, x_ref: NDArray, x: NDArray, count: int) -> NDArray:
         x_updated = np.concatenate([x_ref, x], axis=0)
         return x_updated[-self.n :]
 
@@ -78,10 +137,10 @@ class ReservoirSamplingUpdate(UpdateStrategy):
     Parameters
     ----------
     n : int
-        Update with reservoir sampling of size n.
+        Update with last n instances seen by the detector.
     """
 
-    def __call__(self, x_ref: np.ndarray, x: np.ndarray, count: int) -> np.ndarray:
+    def __call__(self, x_ref: NDArray, x: NDArray, count: int) -> NDArray:
         if x.shape[0] + count <= self.n:
             return np.concatenate([x_ref, x], axis=0)
 
@@ -103,15 +162,64 @@ class ReservoirSamplingUpdate(UpdateStrategy):
 
 
 class BaseDrift:
-    """Generic drift detector component handling preprocessing of data and correction"""
+    """
+    A generic drift detection component for preprocessing data and applying statistical correction.
+
+    This class handles common tasks related to drift detection, such as preprocessing
+    the reference data (`x_ref`), performing statistical correction (e.g., Bonferroni, FDR),
+    and updating the reference data if needed.
+
+    Parameters
+    ----------
+    x_ref : ArrayLike
+        The reference dataset used for drift detection. This is the baseline data against
+        which new data points will be compared.
+    p_val : float, optional
+        The significance level for detecting drift, by default 0.05.
+    x_ref_preprocessed : bool, optional
+        Flag indicating whether the reference data has already been preprocessed, by default False.
+    update_x_ref : UpdateStrategy, optional
+        A strategy object specifying how the reference data should be updated when drift is detected,
+        by default None.
+    preprocess_fn : Callable[[ArrayLike], ArrayLike], optional
+        A function to preprocess the data before drift detection, by default None.
+    correction : {'bonferroni', 'fdr'}, optional
+        Statistical correction method applied to p-values, by default "bonferroni".
+
+    Attributes
+    ----------
+    _x_ref : ArrayLike
+        The reference dataset that is either raw or preprocessed.
+    p_val : float
+        The significance level for drift detection.
+    update_x_ref : UpdateStrategy or None
+        The strategy for updating the reference data if applicable.
+    preprocess_fn : Callable or None
+        Function used for preprocessing input data before drift detection.
+    correction : str
+        Statistical correction method applied to p-values.
+    n : int
+        The number of samples in the reference dataset (`x_ref`).
+    x_ref_preprocessed : bool
+        A flag that indicates whether the reference dataset has been preprocessed.
+    _x_refcount : int
+        Counter for how many times the reference data has been accessed after preprocessing.
+
+    Methods
+    -------
+    x_ref:
+        Property that returns the reference dataset, and applies preprocessing if not already done.
+    _preprocess(x):
+        Preprocesses the given data using the specified `preprocess_fn` if provided.
+    """
 
     def __init__(
         self,
         x_ref: ArrayLike,
         p_val: float = 0.05,
         x_ref_preprocessed: bool = False,
-        update_x_ref: Optional[UpdateStrategy] = None,
-        preprocess_fn: Optional[Callable[[ArrayLike], ArrayLike]] = None,
+        update_x_ref: UpdateStrategy | None = None,
+        preprocess_fn: Callable[[ArrayLike], ArrayLike] | None = None,
         correction: Literal["bonferroni", "fdr"] = "bonferroni",
     ) -> None:
         # Type checking
@@ -136,7 +244,15 @@ class BaseDrift:
         self._x_refcount = 0
 
     @property
-    def x_ref(self) -> np.ndarray:
+    def x_ref(self) -> NDArray:
+        """
+        Retrieve the reference data, applying preprocessing if not already done.
+
+        Returns
+        -------
+        NDArray
+            The reference dataset (`x_ref`), preprocessed if needed.
+        """
         if not self.x_ref_preprocessed:
             self.x_ref_preprocessed = True
             if self.preprocess_fn is not None:
@@ -146,18 +262,75 @@ class BaseDrift:
         return self._x_ref
 
     def _preprocess(self, x: ArrayLike) -> ArrayLike:
-        """Data preprocessing before computing the drift scores."""
+        """
+        Preprocess the given data before computing the drift scores.
+
+        Parameters
+        ----------
+        x : ArrayLike
+            The input data to preprocess.
+
+        Returns
+        -------
+        ArrayLike
+            The preprocessed input data.
+        """
         if self.preprocess_fn is not None:
             x = self.preprocess_fn(x)
         return x
 
 
-class BaseUnivariateDrift(BaseDrift):
+class BaseDriftUnivariate(BaseDrift):
     """
-    Generic drift detector component which serves as a base class for methods using
-    univariate tests. If n_features > 1, a multivariate correction is applied such
-    that the false positive rate is upper bounded by the specified p-value, with
-    equality in the case of independent features.
+    Base class for drift detection methods using univariate statistical tests.
+
+    This class inherits from `BaseDrift` and serves as a generic component for detecting
+    distribution drift in univariate features. If the number of features `n_features` is greater
+    than 1, a multivariate correction method (e.g., Bonferroni or FDR) is applied to control
+    the false positive rate, ensuring it does not exceed the specified p-value.
+
+    Parameters
+    ----------
+    x_ref : ArrayLike
+        Reference data used as the baseline to compare against when detecting drift.
+    p_val : float, default 0.05
+        Significance level used for detecting drift.
+    x_ref_preprocessed : bool, default False
+        Indicates whether the reference data has been preprocessed.
+    update_x_ref : UpdateStrategy | None, default None
+        Strategy for updating the reference data when drift is detected.
+    preprocess_fn : Callable[ArrayLike] | None, default None
+        Function used to preprocess input data before detecting drift.
+    correction : 'bonferroni' | 'fdr', default 'bonferroni'
+        Multivariate correction method applied to p-values.
+    n_features : int | None, default None
+        Number of features used in the univariate drift tests. If not provided, it will
+        be inferred from the data.
+
+    Attributes
+    ----------
+    _n_features : int | None
+        Number of features in the data. If not provided, it is lazily inferred from the
+        input data and any preprocessing function.
+    p_val : float
+        The significance level for drift detection.
+    correction : str
+        The method for controlling the false discovery rate or applying a Bonferroni correction.
+    update_x_ref : UpdateStrategy | None
+        Strategy for updating the reference data if applicable.
+    preprocess_fn : Callable | None
+        Function used for preprocessing input data before drift detection.
+
+    Methods
+    -------
+    n_features:
+        Property that returns the number of features, inferring it if necessary.
+    score(x):
+        Abstract method to compute univariate feature scores after preprocessing.
+    _apply_correction(p_vals):
+        Apply a statistical correction to p-values to account for multiple testing.
+    predict(x):
+        Predict whether drift has occurred on a batch of data, applying multivariate correction if needed.
     """
 
     def __init__(
@@ -165,10 +338,10 @@ class BaseUnivariateDrift(BaseDrift):
         x_ref: ArrayLike,
         p_val: float = 0.05,
         x_ref_preprocessed: bool = False,
-        update_x_ref: Optional[UpdateStrategy] = None,
-        preprocess_fn: Optional[Callable[[ArrayLike], ArrayLike]] = None,
+        update_x_ref: UpdateStrategy | None = None,
+        preprocess_fn: Callable[[ArrayLike], ArrayLike] | None = None,
         correction: Literal["bonferroni", "fdr"] = "bonferroni",
-        n_features: Optional[int] = None,
+        n_features: int | None = None,
     ) -> None:
         super().__init__(
             x_ref,
@@ -183,6 +356,18 @@ class BaseUnivariateDrift(BaseDrift):
 
     @property
     def n_features(self) -> int:
+        """
+        Get the number of features in the reference data.
+
+        If the number of features is not provided during initialization, it will be inferred
+        from the reference data (``x_ref``). If a preprocessing function is provided, the number
+        of features will be inferred after applying the preprocessing function.
+
+        Returns
+        -------
+        int
+            Number of features in the reference data.
+        """
         # lazy process n_features as needed
         if not isinstance(self._n_features, int):
             # compute number of features for the univariate tests
@@ -198,13 +383,43 @@ class BaseUnivariateDrift(BaseDrift):
 
     @preprocess_x
     @abstractmethod
-    def score(self, x: ArrayLike) -> Tuple[np.ndarray, np.ndarray]:
-        """Abstract method to calculate feature score after preprocessing"""
+    def score(self, x: ArrayLike) -> tuple[NDArray[np.float32], NDArray[np.float32]]:
+        """
+        Abstract method to calculate feature scores after preprocessing.
+
+        Parameters
+        ----------
+        x : ArrayLike
+            The batch of data to calculate univariate drift scores for each feature.
 
-    def _apply_correction(self, p_vals: np.ndarray) -> Tuple[int, float]:
+        Returns
+        -------
+        tuple[NDArray, NDArray]
+            A tuple containing p-values and distance statistics for each feature.
+        """
+
+    def _apply_correction(self, p_vals: NDArray) -> tuple[bool, float]:
+        """
+        Apply the specified correction method (Bonferroni or FDR) to the p-values.
+
+        If the correction method is Bonferroni, the threshold for detecting drift
+        is divided by the number of features. For FDR, the correction is applied
+        using the Benjamini-Hochberg procedure.
+
+        Parameters
+        ----------
+        p_vals : NDArray
+            Array of p-values from the univariate tests for each feature.
+
+        Returns
+        -------
+        tuple[bool, float]
+            A tuple containing a boolean indicating if drift was detected and the
+            threshold after correction.
+        """
         if self.correction == "bonferroni":
             threshold = self.p_val / self.n_features
-            drift_pred = int((p_vals < threshold).any())
+            drift_pred = bool((p_vals < threshold).any())
             return drift_pred, threshold
         elif self.correction == "fdr":
             n = p_vals.shape[0]
@@ -215,18 +430,18 @@ class BaseUnivariateDrift(BaseDrift):
             try:
                 idx_threshold = int(np.where(below_threshold)[0].max())
             except ValueError:  # sorted p-values not below thresholds
-                return int(below_threshold.any()), q_threshold.min()
-            return int(below_threshold.any()), q_threshold[idx_threshold]
+                return bool(below_threshold.any()), q_threshold.min()
+            return bool(below_threshold.any()), q_threshold[idx_threshold]
         else:
             raise ValueError("`correction` needs to be either `bonferroni` or `fdr`.")
 
+    @set_metadata("dataeval.detectors")
     @preprocess_x
     @update_x_ref
     def predict(
         self,
         x: ArrayLike,
-        drift_type: Literal["batch", "feature"] = "batch",
-    ) -> Dict[str, Union[int, float, np.ndarray]]:
+    ) -> DriftOutput:
         """
         Predict whether a batch of data has drifted from the reference data and update
         reference data using specified update strategy.
@@ -235,34 +450,16 @@ class BaseUnivariateDrift(BaseDrift):
         ----------
         x : ArrayLike
             Batch of instances.
-        drift_type : Literal["batch", "feature"], default "batch"
-            Predict drift at the 'feature' or 'batch' level. For 'batch', the test
-            statistics for each feature are aggregated using the Bonferroni or False
-            Discovery Rate correction (if n_features>1).
 
         Returns
         -------
-        Dictionary containing the drift prediction and optionally the feature level
-                p-values, threshold after multivariate correction if needed and test
-                statistics.
+        DriftOutput
+            Dictionary containing the drift prediction and optionally the feature level
+            p-values, threshold after multivariate correction if needed and test statistics.
         """
         # compute drift scores
         p_vals, dist = self.score(x)
 
-        # TODO: return both feature-level and batch-level drift predictions by default
-        # values below p-value threshold are drift
-        if drift_type == "feature":
-            drift_pred = (p_vals < self.p_val).astype(int)
-            threshold = self.p_val
-        elif drift_type == "batch":
-            drift_pred, threshold = self._apply_correction(p_vals)
-        else:
-            raise ValueError("`drift_type` needs to be either `feature` or `batch`.")
-
-        # populate drift dict
-        return {
-            "is_drift": drift_pred,
-            "p_val": p_vals,
-            "threshold": threshold,
-            "distance": dist,
-        }
+        feature_drift = (p_vals < self.p_val).astype(np.bool_)
+        drift_pred, threshold = self._apply_correction(p_vals)
+        return DriftOutput(drift_pred, threshold, feature_drift, self.p_val, p_vals, dist)

dataeval/_internal/detectors/drift/cvm.py

@@ -6,50 +6,51 @@ Original code Copyright (c) 2023 Seldon Technologies Ltd
 Licensed under Apache Software License (Apache 2.0)
 """
 
-from typing import Callable, Literal, Optional, Tuple
+from __future__ import annotations
+
+from typing import Callable, Literal
 
 import numpy as np
-from numpy.typing import ArrayLike
+from numpy.typing import ArrayLike, NDArray
 from scipy.stats import cramervonmises_2samp
 
 from dataeval._internal.interop import to_numpy
 
-from .base import BaseUnivariateDrift, UpdateStrategy, preprocess_x
+from .base import BaseDriftUnivariate, UpdateStrategy, preprocess_x
 
 
-class DriftCVM(BaseUnivariateDrift):
+class DriftCVM(BaseDriftUnivariate):
     """
-    Cramér-von Mises (CVM) data drift detector, which tests for any change in the
-    distribution of continuous univariate data. For multivariate data, a separate
-    CVM test is applied to each feature, and the obtained p-values are aggregated
-    via the Bonferroni or False Discovery Rate (FDR) corrections.
+    Drift detector employing the Cramér-von Mises (CVM) distribution test.
+
+    The CVM test detects changes in the distribution of continuous
+    univariate data. For multivariate data, a separate CVM test is applied to each
+    feature, and the obtained p-values are aggregated via the Bonferroni or
+    False Discovery Rate (FDR) corrections.
 
     Parameters
     ----------
     x_ref : ArrayLike
         Data used as reference distribution.
-    p_val : float, default 0.05
+    p_val : float | None, default 0.05
         p-value used for significance of the statistical test for each feature.
         If the FDR correction method is used, this corresponds to the acceptable
         q-value.
     x_ref_preprocessed : bool, default False
-        Whether the given reference data `x_ref` has been preprocessed yet. If
-        `x_ref_preprocessed=True`, only the test data `x` will be preprocessed at
-        prediction time. If `x_ref_preprocessed=False`, the reference data will also
-        be preprocessed.
-    update_x_ref : Optional[UpdateStrategy], default None
+        Whether the given reference data ``x_ref`` has been preprocessed yet.
+        If ``True``, only the test data ``x`` will be preprocessed at prediction time.
+        If ``False``, the reference data will also be preprocessed.
+    update_x_ref : UpdateStrategy | None, default None
         Reference data can optionally be updated using an UpdateStrategy class. Update
-        using the last n instances seen by the detector with
-        :py:class:`dataeval.detectors.LastSeenUpdateStrategy`
-        or via reservoir sampling with
-        :py:class:`dataeval.detectors.ReservoirSamplingUpdateStrategy`.
-    preprocess_fn : Optional[Callable[[ArrayLike], ArrayLike]], default None
+        using the last n instances seen by the detector with LastSeenUpdateStrategy
+        or via reservoir sampling with ReservoirSamplingUpdateStrategy.
+    preprocess_fn : Callable | None, default None
         Function to preprocess the data before computing the data drift metrics.
         Typically a dimensionality reduction technique.
-    correction : Literal["bonferroni", "fdr"], default "bonferroni"
+    correction : "bonferroni" | "fdr", default "bonferroni"
         Correction type for multivariate data. Either 'bonferroni' or 'fdr' (False
         Discovery Rate).
-    n_features
+    n_features : int | None, default None
         Number of features used in the statistical test. No need to pass it if no
         preprocessing takes place. In case of a preprocessing step, this can also
         be inferred automatically but could be more expensive to compute.
@@ -60,10 +61,10 @@ class DriftCVM(BaseUnivariateDrift):
         x_ref: ArrayLike,
         p_val: float = 0.05,
         x_ref_preprocessed: bool = False,
-        update_x_ref: Optional[UpdateStrategy] = None,
-        preprocess_fn: Optional[Callable[[ArrayLike], ArrayLike]] = None,
+        update_x_ref: UpdateStrategy | None = None,
+        preprocess_fn: Callable[[ArrayLike], ArrayLike] | None = None,
         correction: Literal["bonferroni", "fdr"] = "bonferroni",
-        n_features: Optional[int] = None,
+        n_features: int | None = None,
     ) -> None:
         super().__init__(
             x_ref=x_ref,
@@ -76,7 +77,7 @@ class DriftCVM(BaseUnivariateDrift):
         )
 
     @preprocess_x
-    def score(self, x: ArrayLike) -> Tuple[np.ndarray, np.ndarray]:
+    def score(self, x: ArrayLike) -> tuple[NDArray[np.float32], NDArray[np.float32]]:
         """
         Performs the two-sample Cramér-von Mises test(s), computing the p-value and
         test statistic per feature.
@@ -88,7 +89,8 @@ class DriftCVM(BaseUnivariateDrift):
 
         Returns
         -------
-        Feature level p-values and CVM statistics.
+        tuple[NDArray, NDArray]
+            Feature level p-values and CVM statistic
         """
         x_np = to_numpy(x)
         x_np = x_np.reshape(x_np.shape[0], -1)

dataeval/_internal/detectors/drift/ks.py

@@ -6,51 +6,53 @@ Original code Copyright (c) 2023 Seldon Technologies Ltd
 Licensed under Apache Software License (Apache 2.0)
 """
 
-from typing import Callable, Literal, Optional, Tuple
+from __future__ import annotations
+
+from typing import Callable, Literal
 
 import numpy as np
-from numpy.typing import ArrayLike
+from numpy.typing import ArrayLike, NDArray
 from scipy.stats import ks_2samp
 
 from dataeval._internal.interop import to_numpy
 
-from .base import BaseUnivariateDrift, UpdateStrategy, preprocess_x
+from .base import BaseDriftUnivariate, UpdateStrategy, preprocess_x
 
 
-class DriftKS(BaseUnivariateDrift):
+class DriftKS(BaseDriftUnivariate):
     """
-    Kolmogorov-Smirnov (K-S) data drift detector with Bonferroni or False Discovery
-    Rate (FDR) correction for multivariate data.
+    Drift detector employing the Kolmogorov-Smirnov (KS) distribution test.
+
+    The KS test detects changes in the maximum distance between two data
+    distributions with Bonferroni or False Discovery Rate (FDR) correction
+    for multivariate data.
 
     Parameters
     ----------
-    x_ref : np.ndarray
+    x_ref : ArrayLike
         Data used as reference distribution.
-    p_val : float, default 0.05
+    p_val : float | None, default 0.05
         p-value used for significance of the statistical test for each feature.
         If the FDR correction method is used, this corresponds to the acceptable
         q-value.
     x_ref_preprocessed : bool, default False
-        Whether the given reference data `x_ref` has been preprocessed yet. If
-        `x_ref_preprocessed=True`, only the test data `x` will be preprocessed at
-        prediction time. If `x_ref_preprocessed=False`, the reference data will also
-        be preprocessed.
-    update_x_ref : Optional[UpdateStrategy], default None
+        Whether the given reference data ``x_ref`` has been preprocessed yet.
+        If ``True``, only the test data ``x`` will be preprocessed at prediction time.
+        If ``False``, the reference data will also be preprocessed.
+    update_x_ref : UpdateStrategy | None, default None
         Reference data can optionally be updated using an UpdateStrategy class. Update
-        using the last n instances seen by the detector with
-        :py:class:`dataeval.detectors.LastSeenUpdateStrategy`
-        or via reservoir sampling with
-        :py:class:`dataeval.detectors.ReservoirSamplingUpdateStrategy`.
-    preprocess_fn : Optional[Callable[[np.ndarray], np.ndarray]], default None
+        using the last n instances seen by the detector with LastSeenUpdateStrategy
+        or via reservoir sampling with ReservoirSamplingUpdateStrategy.
+    preprocess_fn : Callable | None, default None
         Function to preprocess the data before computing the data drift metrics.
         Typically a dimensionality reduction technique.
-    correction : Literal["bonferroni", "fdr"], default "bonferroni"
+    correction : "bonferroni" | "fdr", default "bonferroni"
         Correction type for multivariate data. Either 'bonferroni' or 'fdr' (False
         Discovery Rate).
-    alternative : Literal["two-sided", "less", "greater"], default "two-sided"
+    alternative : "two-sided" | "less" | "greater", default "two-sided"
         Defines the alternative hypothesis. Options are 'two-sided', 'less' or
         'greater'.
-    n_features
+    n_features : int | None, default None
         Number of features used in the statistical test. No need to pass it if no
         preprocessing takes place. In case of a preprocessing step, this can also
         be inferred automatically but could be more expensive to compute.
@@ -61,11 +63,11 @@ class DriftKS(BaseUnivariateDrift):
         x_ref: ArrayLike,
         p_val: float = 0.05,
         x_ref_preprocessed: bool = False,
-        update_x_ref: Optional[UpdateStrategy] = None,
-        preprocess_fn: Optional[Callable[[ArrayLike], ArrayLike]] = None,
+        update_x_ref: UpdateStrategy | None = None,
+        preprocess_fn: Callable[[ArrayLike], ArrayLike] | None = None,
         correction: Literal["bonferroni", "fdr"] = "bonferroni",
         alternative: Literal["two-sided", "less", "greater"] = "two-sided",
-        n_features: Optional[int] = None,
+        n_features: int | None = None,
     ) -> None:
         super().__init__(
             x_ref=x_ref,
@@ -81,18 +83,19 @@ class DriftKS(BaseUnivariateDrift):
         self.alternative = alternative
 
     @preprocess_x
-    def score(self, x: ArrayLike) -> Tuple[np.ndarray, np.ndarray]:
+    def score(self, x: ArrayLike) -> tuple[NDArray[np.float32], NDArray[np.float32]]:
         """
-        Compute K-S scores and statistics per feature.
+        Compute KS scores and statistics per feature.
 
         Parameters
         ----------
-        x
+        x : ArrayLike
             Batch of instances.
 
         Returns
         -------
-        Feature level p-values and K-S statistics.
+        tuple[NDArray, NDArray]
+            Feature level p-values and KS statistic
         """
         x = to_numpy(x)
         x = x.reshape(x.shape[0], -1)