chemotools 0.1.8__py3-none-any.whl → 0.1.10__py3-none-any.whl

chemotools/augmentation/_index_shift.py

@@ -1,23 +1,37 @@
 from typing import Literal, Optional
 
 import numpy as np
-from numpy.polynomial import polynomial as poly
+from scipy.signal import convolve
+from scipy import stats
 from sklearn.base import BaseEstimator, TransformerMixin, OneToOneFeatureMixin
 from sklearn.utils.validation import check_is_fitted, validate_data
 
 
 class IndexShift(TransformerMixin, OneToOneFeatureMixin, BaseEstimator):
     """
-    Shift the spectrum a given number of indices between - shift and + shift drawn
+    Shift the spectrum a given number of indices between -shift and +shift drawn
     from a discrete uniform distribution.
 
     Parameters
     ----------
-    shift : float, default=0.0
-        Shifts the data by a random integer between -shift and shift.
+    shift : int, default=0
+        Maximum number of indices by which the data is randomly shifted.
+        The actual shift is a random integer between -shift and shift (inclusive).
 
-    random_state : int, default=None
-        The random state to use for the random number generator.
+    padding_mode : {'zeros', 'constant', 'wrap', 'extend', 'mirror', 'linear'}, default='linear'
+        Specifies how to handle padding when shifting the data:
+            - 'zeros': Pads with zeros.
+            - 'constant': Pads with a constant value defined by `pad_value`.
+            - 'wrap': Circular shift (wraps around).
+            - 'extend': Extends using edge values.
+            - 'mirror': Mirrors the signal.
+            - 'linear': Uses linear regression to extrapolate values.
+
+    pad_value : float, default=0.0
+        The value used for padding when `padding_mode='constant'`.
+
+    random_state : int, optional, default=None
+        The random seed for reproducibility.
 
     Attributes
     ----------
@@ -27,23 +41,22 @@ class IndexShift(TransformerMixin, OneToOneFeatureMixin, BaseEstimator):
     _is_fitted : bool
         Whether the transformer has been fitted to data.
 
-    Methods
-    -------
-    fit(X, y=None)
-        Fit the transformer to the input data.
-
-    transform(X, y=0, copy=True)
-        Transform the input data by shifting the spectrum.
+    _rng : numpy.random.Generator
+        Random number generator instance used for shifting.
     """
 
     def __init__(
         self,
         shift: int = 0,
-        fill_method: Literal["constant", "linear", "quadratic"] = "constant",
+        padding_mode: Literal[
+            "zeros", "constant", "wrap", "extend", "mirror", "linear"
+        ] = "linear",
+        pad_value: float = 0.0,
         random_state: Optional[int] = None,
     ):
         self.shift = shift
-        self.fill_method = fill_method
+        self.padding_mode = padding_mode
+        self.pad_value = pad_value
         self.random_state = random_state
 
     def fit(self, X: np.ndarray, y=None) -> "IndexShift":
@@ -68,12 +81,6 @@ class IndexShift(TransformerMixin, OneToOneFeatureMixin, BaseEstimator):
             self, X, y="no_validation", ensure_2d=True, reset=True, dtype=np.float64
         )
 
-        # Set the number of features
-        self.n_features_in_ = X.shape[1]
-
-        # Set the fitted attribute to True
-        self._is_fitted = True
-
         # Instantiate the random number generator
         self._rng = np.random.default_rng(self.random_state)
 
@@ -94,10 +101,10 @@ class IndexShift(TransformerMixin, OneToOneFeatureMixin, BaseEstimator):
         Returns
         -------
         X_ : np.ndarray of shape (n_samples, n_features)
-            The transformed data.
+            The transformed data with the applied shifts.
         """
         # Check that the estimator is fitted
-        check_is_fitted(self, "_is_fitted")
+        check_is_fitted(self, "n_features_in_")
 
         # Check that X is a 2D array and has only finite values
         X_ = validate_data(
@@ -110,90 +117,98 @@ class IndexShift(TransformerMixin, OneToOneFeatureMixin, BaseEstimator):
             dtype=np.float64,
         )
 
-        # Check that the number of features is the same as the fitted data
-        if X_.shape[1] != self.n_features_in_:
-            raise ValueError(
-                f"Expected {self.n_features_in_} features but got {X_.shape[1]}"
-            )
-
         # Calculate the standard normal variate
         for i, x in enumerate(X_):
-            X_[i] = self._shift_vector(x)
+            X_[i] = self._shift_signal(x)
 
         return X_.reshape(-1, 1) if X_.ndim == 1 else X_
 
-    def _shift_spectrum(self, x) -> np.ndarray:
-        shift_amount = self._rng.integers(-self.shift, self.shift, endpoint=True)
-        return np.roll(x, shift_amount)
-
-    def _shift_vector(
-        self,
-        x: np.ndarray,
-    ) -> np.ndarray:
+    def _shift_signal(self, x: np.ndarray):
         """
-        Shift vector with option to fill missing values.
-
-        Args:
-            arr: Input numpy array
-            shift: Number of positions to shift
-            fill_method: Method to fill missing values
-                'constant': fill with first/last value
-                'linear': fill using linear regression
-                'quadratic': fill using quadratic regression
-
-        Returns:
-            Shifted numpy array
-        """
-        shift = self._rng.integers(-self.shift, self.shift, endpoint=True)
-
-        result = np.roll(x, shift)
-
-        if self.fill_method == "constant":
-            if shift > 0:
-                result[:shift] = x[0]
-            elif shift < 0:
-                result[shift:] = x[-1]
-
-        elif self.fill_method == "linear":
-            if shift > 0:
-                x_ = np.arange(5)
-                coeffs = poly.polyfit(x_, x[:5], 1)
+        Shifts a discrete signal using convolution with a Dirac delta kernel.
 
-                extrapolate_x = np.arange(-shift, 0)
-                extrapolated_values = poly.polyval(extrapolate_x, coeffs)
-
-                result[:shift] = extrapolated_values
-
-            elif shift < 0:
-                x_ = np.arange(5)
-                coeffs = poly.polyfit(x_, x[-5:], 1)
-
-                extrapolate_x = np.arange(len(x_), len(x_) - shift)
-                extrapolated_values = poly.polyval(extrapolate_x, coeffs)
-
-                result[shift:] = extrapolated_values
-
-        elif self.fill_method == "quadratic":
-            if shift > 0:
-                # Use first 3 values for quadratic regression
-                x_ = np.arange(5)
-                coeffs = poly.polyfit(x_, x[:5], 2)
-
-                # Extrapolate to fill shifted region
-                extrapolate_x = np.arange(-shift, 0)
-                extrapolated_values = poly.polyval(extrapolate_x, coeffs)
-
-                result[:shift] = extrapolated_values
-
-            elif shift < 0:
-                # Use last 3 values for quadratic regression
-                x_ = np.arange(5)
-                coeffs = poly.polyfit(x_, x[-5:], 2)
-
-                # Extrapolate to fill shifted region
-                extrapolate_x = np.arange(len(x_), len(x_) - shift)
-                extrapolated_values = poly.polyval(extrapolate_x, coeffs)
+        Parameters
+        ----------
+        x : np.ndarray of shape (n_features,)
+            The input signal to shift.
 
-                result[shift:] = extrapolated_values
+        Returns
+        -------
+        result : np.ndarray of shape (n_features,)
+            The shifted signal.
+        """
+        shift = self._rng.integers(-self.shift, self.shift, endpoint=True)
 
-        return result
+        if self.padding_mode == "wrap":
+            return np.roll(x, shift)
+
+        # Create Dirac delta kernel with proper dimensions
+
+        if shift >= 0:
+            kernel = np.zeros(shift + 1)
+            kernel[-1] = 1
+        else:
+            kernel = np.zeros(-shift + 1)
+            kernel[0] = 1
+
+        # Convolve signal with kernel
+        shifted = convolve(x, kernel, mode="full")
+
+        if shift >= 0:
+            result = shifted[: len(x)] if x.ndim == 1 else shifted[: x.shape[0]]
+            pad_length = shift
+            pad_left = True
+        else:
+            result = shifted[-len(x) :] if x.ndim == 1 else shifted[-x.shape[0] :]
+            pad_length = -shift
+            pad_left = False
+
+        if self.padding_mode == "zeros":
+            return result
+
+        elif self.padding_mode == "constant":
+            mask = np.abs(result) < 1e-10
+            result[mask] = self.pad_value
+            return result
+
+        elif self.padding_mode == "mirror":
+            if pad_left:
+                pad_values = x[pad_length - 1 :: -1]
+                result[:pad_length] = pad_values[-pad_length:]
+            else:
+                pad_values = x[:-1][::-1]
+                result[-pad_length:] = pad_values[:pad_length]
+
+            return result
+
+        elif self.padding_mode == "extend":
+            if pad_left:
+                result[:pad_length] = x[0]
+            else:
+                result[-pad_length:] = x[-1]
+            return result
+
+        elif self.padding_mode == "linear":
+            # Get points for linear regression
+            if pad_left:
+                points = x[: pad_length + 1]  # Take first pad_length+1 points
+                x_coords = np.arange(len(points))
+                slope, intercept, _, _, _ = stats.linregress(x_coords, points)
+
+                # Generate new points using linear regression
+                new_x = np.arange(-pad_length, 0)
+                extrapolated = slope * new_x + intercept
+                result[:pad_length] = extrapolated
+            else:
+                points = x[-pad_length - 1 :]  # Take last pad_length+1 points
+                x_coords = np.arange(len(points))
+                slope, intercept, _, _, _ = stats.linregress(x_coords, points)
+
+                # Generate new points using linear regression
+                new_x = np.arange(len(points), len(points) + pad_length)
+                extrapolated = slope * new_x + intercept
+                result[-pad_length:] = extrapolated
+            return result
+
+        else:
+            raise ValueError(f"Unknown padding mode: {self.padding_mode}")

chemotools/outliers/__init__.py

@@ -0,0 +1,7 @@
+from .dmodx import DModX
+from .hotelling_t2 import HotellingT2
+from .q_residuals import QResiduals
+from .leverage import Leverage
+from .studentized_residuals import StudentizedResiduals
+
+__all__ = ["DModX", "HotellingT2", "QResiduals", "Leverage", "StudentizedResiduals"]

chemotools/outliers/_base.py

@@ -0,0 +1,180 @@
+from abc import ABC, abstractmethod
+from typing import Union, Optional
+
+import numpy as np
+
+from sklearn.base import BaseEstimator, OutlierMixin
+from sklearn.decomposition._base import _BasePCA
+from sklearn.cross_decomposition._pls import _PLS
+from sklearn.pipeline import Pipeline
+from sklearn.utils.validation import check_is_fitted
+
+from ._utils import validate_confidence, validate_and_extract_model
+
+ModelTypes = Union[_BasePCA, _PLS]
+
+
+class _ModelResidualsBase(ABC, BaseEstimator, OutlierMixin):
+    """Base class for model outlier calculations.
+
+    Implements statistical calculations for outlier detection in dimensionality
+    reduction models like PCA and PLS.
+
+    Parameters
+    ----------
+    model : Union[ModelTypes, Pipeline]
+        A fitted _BasePCA or _PLS models or Pipeline ending with such a model
+    confidence : float
+        Confidence level for statistical calculations (between 0 and 1)
+
+    Attributes
+    ----------
+    model_ : ModelTypes
+        The fitted model of type _BasePCA or _PLS
+
+    preprocessing_ : Optional[Pipeline]
+        Preprocessing steps before the model
+
+    n_features_in_ : int
+        Number of features in the input data
+
+    n_components_ : int
+        Number of components in the model
+
+    n_samples_ : int
+        Number of samples used to train the model
+
+    critical_value_ : float
+        The calculated critical value for outlier detection
+    """
+
+    def __init__(
+        self,
+        model: Union[ModelTypes, Pipeline],
+        confidence: float,
+    ) -> None:
+        (
+            self.model_,
+            self.preprocessing_,
+            self.n_features_in_,
+            self.n_components_,
+            self.n_samples_,
+        ) = validate_and_extract_model(model)
+        self.confidence = validate_confidence(confidence)
+
+    def fit_predict_residuals(
+        self, X: np.ndarray, y: Optional[np.ndarray] = None
+    ) -> np.ndarray:
+        """Fit the model to the input data and calculate the residuals.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Input data
+
+        y : array-like of shape (n_samples,), default=None
+            Target values
+
+        Returns
+        -------
+        ndarray of shape (n_samples,)
+            The residuals of the model
+        """
+        self.fit(X, y)
+        return self.predict_residuals(X, y, validate=True)
+
+    @abstractmethod
+    def predict_residuals(
+        self, X: np.ndarray, y: Optional[np.ndarray], validate: bool
+    ) -> np.ndarray:
+        """Calculate the residuals of the model.
+
+        Returns
+        -------
+        ndarray of shape (n_samples,)
+            The residuals of the model
+        """
+
+    @abstractmethod
+    def _calculate_critical_value(self, X: Optional[np.ndarray]) -> float:
+        """Calculate the critical value for outlier detection.
+
+        Returns
+        -------
+        float
+            The calculated critical value for outlier detection
+        """
+
+
+class _ModelDiagnosticsBase(ABC):
+    """Base class for model diagnostics methods. This does not implement outlier detection algorithms,
+    but rather implements methods that are used to assess trained models.
+
+    Parameters
+    ----------
+    model : Union[ModelTypes, Pipeline]
+        A fitted PCA/PLS model or Pipeline ending with such a model
+
+    Attributes
+    ----------
+    model_ : ModelTypes
+        The fitted model of type _BasePCA or _PLS
+
+    preprocessing_ : Optional[Pipeline]
+        Preprocessing steps before the model
+
+    """
+
+    def __init__(self, model: Union[ModelTypes, Pipeline]):
+        self.model_, self.preprocessing_ = self._validate_and_extract_model(model)
+
+    def _validate_and_extract_model(self, model):
+        """Validate and extract the model and preprocessing steps.
+
+        Parameters
+        ----------
+        model : Union[ModelTypes, Pipeline]
+            A fitted PCA/PLS model or Pipeline ending with such a model
+
+        Returns
+        -------
+        Tuple[ModelTypes, Optional[Pipeline]]
+            The extracted model and preprocessing steps
+
+        Raises
+        ------
+        ValueError
+            If the model is not of type _BasePCA or _PLS or a Pipeline ending with one of these types or if the model is not fitted
+        """
+        if isinstance(model, Pipeline):
+            preprocessing = model[:-1]
+            model = model[-1]
+        else:
+            preprocessing = None
+
+        if isinstance(model, (_BasePCA, _PLS)):
+            check_is_fitted(model)
+        else:
+            raise ValueError(
+                "Model not a valid model. Must be of base type _BasePCA or _PLS or a Pipeline ending with one of these types."
+            )
+        check_is_fitted(model)
+        return model, preprocessing
+
+    @abstractmethod
+    def predict(self, X: np.ndarray, y: Optional[np.ndarray]) -> np.ndarray:
+        """Predict the output of the model.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Input data
+
+        y : array-like of shape (n_samples,), default=None
+            Target values
+
+        Returns
+        -------
+        ndarray of shape (n_samples,)
+            Predicted values
+        """

chemotools/outliers/_utils.py

@@ -0,0 +1,91 @@
+from typing import Optional, Tuple, Union
+
+from sklearn.cross_decomposition._pls import _PLS
+from sklearn.decomposition._base import _BasePCA
+from sklearn.pipeline import Pipeline
+from sklearn.utils.validation import check_is_fitted
+
+ModelTypes = Union[_BasePCA, _PLS]
+
+
+def get_model_parameters(model: ModelTypes) -> Tuple[int, int, int]:
+    """
+    Get the number of features, components and samples from a model with PLS or PCA. types.
+
+    Parameters
+    ----------
+    model : ModelType
+        A fitted model of type _BasePCA or _PLS
+
+    Returns
+    -------
+    Tuple[int, int, int]
+        The number of features, components and samples in the model
+    """
+    if isinstance(model, _BasePCA):
+        return model.n_features_in_, model.n_components_, model.n_samples_
+    elif isinstance(model, _PLS):
+        return model.n_features_in_, model.n_components, len(model.x_scores_)
+    else:
+        raise ValueError(
+            "Model not a valid model. Must be of base type _BasePCA or _PLS or a Pipeline ending with one of these types."
+        )
+
+
+def validate_confidence(confidence: float) -> float:
+    """Validate parameters using sklearn conventions.
+
+    Parameters
+    ----------
+    confidence : float
+        Confidence level for statistical calculations (between 0 and 1)
+
+    Returns
+    -------
+    float
+        The validated confidence level
+
+    Raises
+    ------
+    ValueError
+        If confidence is not between 0 and 1
+    """
+    if not 0 < confidence < 1:
+        raise ValueError("Confidence must be between 0 and 1")
+    return confidence
+
+
+def validate_and_extract_model(
+    model: Union[ModelTypes, Pipeline],
+) -> Tuple[ModelTypes, Optional[Pipeline], int, int, int]:
+    """Validate and extract the model and preprocessing steps.
+
+    Parameters
+    ----------
+    model : Union[ModelTypes, Pipeline]
+        A fitted PCA/PLS model or Pipeline ending with such a model
+
+    Returns
+    -------
+    Tuple[ModelTypes, Optional[Pipeline]]
+        The extracted model and preprocessing steps
+
+    Raises
+    ------
+    ValueError
+        If the model is not of type _BasePCA or _PLS or a Pipeline ending with one of these types or if the model is not fitted
+    """
+    if isinstance(model, Pipeline):
+        preprocessing = model[:-1]
+        model = model[-1]
+    else:
+        preprocessing = None
+
+    if not isinstance(model, (_BasePCA, _PLS)):
+        raise ValueError(
+            "Model not a valid model. Must be of base type _BasePCA or _PLS or a Pipeline ending with one of these types."
+        )
+
+    check_is_fitted(model)
+    n_features_in, n_components, n_samples = get_model_parameters(model)
+    return model, preprocessing, n_features_in, n_components, n_samples

chemotools/outliers/dmodx.py

@@ -0,0 +1,146 @@
+from typing import Optional, Union
+import numpy as np
+
+from sklearn.pipeline import Pipeline
+from sklearn.utils.validation import validate_data, check_is_fitted
+from scipy.stats import f as f_distribution
+
+
+from ._base import _ModelResidualsBase, ModelTypes
+
+
+class DModX(_ModelResidualsBase):
+    """Calculate Distance to Model (DModX) statistics.
+
+    DModX measures the distance between an observation and the model plane
+    in the X-space, useful for detecting outliers.
+
+    Parameters
+    ----------
+    model : Union[ModelType, Pipeline]
+        A fitted PCA/PLS model or Pipeline ending with such a model
+
+    confidence : float, default=0.95
+        Confidence level for statistical calculations (between 0 and 1)
+
+    Attributes
+    ----------
+    model_ : ModelType
+        The fitted model of type _BasePCA or _PLS
+
+    preprocessing_ : Optional[Pipeline]
+        Preprocessing steps before the model
+
+    n_features_in_ : int
+        Number of features in the input data
+
+    n_components_ : int
+        Number of components in the model
+
+    n_samples_ : int
+        Number of samples used to train the model
+
+    critical_value_ : float
+        The calculated critical value for outlier detection
+    """
+
+    def __init__(
+        self,
+        model: Union[ModelTypes, Pipeline],
+        confidence: float = 0.95,
+    ) -> None:
+        super().__init__(model, confidence)
+
+    def fit(self, X: np.ndarray, y: Optional[np.ndarray] = None) -> "DModX":
+        """
+        Fit the model to the input data.
+
+        This step calculates the critical value for the outlier detection. In the DmodX method,
+        the critical value is not depend on the input data but on the model parameters.
+        """
+        X = validate_data(
+            self, X, y="no_validation", ensure_2d=True, reset=True, dtype=np.float64
+        )
+
+        self.critical_value_ = self._calculate_critical_value()
+        return self
+
+    def predict(self, X: np.ndarray) -> np.ndarray:
+        """Identify outliers in the input data.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Input data
+
+        Returns
+        -------
+        ndarray of shape (n_samples,)
+            Boolean array indicating outliers
+        """
+        # Check the estimator has been fitted
+        check_is_fitted(self, ["critical_value_"])
+
+        # Validate the input data
+        X = validate_data(
+            self, X, y="no_validation", ensure_2d=True, reset=True, dtype=np.float64
+        )
+
+        # Calculate outliers based on the DModX statistics
+        dmodx_values = self.predict_residuals(X, validate=False)
+        return np.where(dmodx_values > self.critical_value_, -1, 1)
+
+    def predict_residuals(
+        self, X: np.ndarray, y: Optional[np.ndarray] = None, validate: bool = True
+    ) -> np.ndarray:
+        """Calculate DModX statistics for input data.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Input data
+
+        validate : bool, default=True
+            Whether to validate the input data
+
+        Returns
+        -------
+        ndarray of shape (n_samples,)
+            DModX statistics for each sample
+        """
+        # Check the estimator has been fitted
+        check_is_fitted(self, ["critical_value_"])
+
+        # Validate the input data
+        if validate:
+            X = validate_data(
+                self, X, y="no_validation", ensure_2d=True, reset=True, dtype=np.float64
+            )
+
+        # Apply preprocessing if available
+        if self.preprocessing_:
+            X = self.preprocessing_.transform(X)
+
+        # Calculate the DModX statistics
+        X_transformed = self.model_.transform(X)
+        X_reconstructed = self.model_.inverse_transform(X_transformed)
+        squared_errors = np.sum((X - X_reconstructed) ** 2, axis=1)
+
+        return np.sqrt(squared_errors / (self.n_features_in_ - self.n_components_))
+
+    def _calculate_critical_value(self, X: Optional[np.ndarray] = None) -> float:
+        """Calculate F-distribution based critical value.
+
+        Returns
+        -------
+        float
+            The critical value for outlier detection
+        """
+
+        dof_numerator = self.n_features_in_ - self.n_components_
+        dof_denominator = self.n_features_in_ - self.n_components_ - 1
+
+        upper_control_limit = f_distribution.ppf(
+            self.confidence, dof_numerator, dof_denominator
+        )
+        return np.sqrt(upper_control_limit)