chemotools 0.0.22__py3-none-any.whl → 0.1.6__py3-none-any.whl

chemotools/scatter/_extended_multiplicative_scatter_correction.py

@@ -0,0 +1,183 @@
+from typing import Optional
+
+import numpy as np
+from sklearn.base import BaseEstimator, TransformerMixin, OneToOneFeatureMixin
+from sklearn.utils.validation import check_is_fitted, validate_data
+
+
+class ExtendedMultiplicativeScatterCorrection(
+    TransformerMixin, OneToOneFeatureMixin, BaseEstimator
+):
+    """Extended multiplicative scatter correction (EMSC) is a preprocessing technique for
+    removing non linear scatter effects from spectra. It is based on fitting a polynomial
+    regression model to the spectrum using a reference spectrum. The reference spectrum
+    can be the mean or median spectrum of a set of spectra or a selected reerence.
+
+    Note that this implementation does not include further extensions of the model using
+    orthogonal subspace models.
+
+    Parameters
+    ----------
+    reference : np.ndarray, optional
+        The reference spectrum to use for the correction. If None, the mean
+        spectrum will be used. The default is None.
+    use_mean : bool, optional
+        Whether to use the mean spectrum as the reference. The default is True.
+    use_median : bool, optional
+        Whether to use the median spectrum as the reference. The default is False.
+    order : int, optional
+        The order of the polynomial to fit to the spectrum. The default is 2.
+    weights : np.ndarray, optional
+        The weights to use for the weighted EMSC. If None, the standard EMSC
+        will be used. The default is None.
+
+
+    Attributes
+    ----------
+    reference_ : np.ndarray
+        The reference spectrum used for the correction.
+
+    References
+    ----------
+    Nils Kristian Afseth, Achim Kohler. Extended multiplicative signal correction
+    in vibrational spectroscopy, a tutorial, doi:10.1016/j.chemolab.2012.03.004
+
+    Valeria Tafintseva et al. Correcting replicate variation in spectroscopic data by machine learning and
+    model-based pre-processing, doi:10.1016/j.chemolab.2021.104350
+    """
+
+    def __init__(
+        self,
+        reference: Optional[np.ndarray] = None,
+        use_mean: bool = True,
+        use_median: bool = False,
+        order: int = 2,
+        weights: Optional[np.ndarray] = None,
+    ):
+        self.reference = reference
+        self.use_mean = use_mean
+        self.use_median = use_median
+        self.order = order
+        self.weights = weights
+
+    def fit(self, X: np.ndarray, y=None) -> "ExtendedMultiplicativeScatterCorrection":
+        """
+        Fit the transformer to the input data. If no reference is provided, the
+        mean or median spectrum will be calculated from the input data.
+
+        Parameters
+        ----------
+        X : np.ndarray of shape (n_samples, n_features)
+            The input data to fit the transformer to.
+
+        y : None
+            Ignored.
+
+        Returns
+        -------
+        self : MultiplicativeScatterCorrection
+            The fitted transformer.
+        """
+        # Check that X is a 2D array and has only finite values
+        X = validate_data(
+            self, X, y="no_validation", ensure_2d=True, reset=True, dtype=np.float64
+        )
+
+        # Check that the length of the reference is the same as the number of features
+        if self.reference is not None:
+            if len(self.reference) != self.n_features_in_:
+                raise ValueError(
+                    f"Expected {self.n_features_in_} features in reference but got {len(self.reference)}"
+                )
+
+        if self.weights is not None:
+            if len(self.weights) != self.n_features_in_:
+                raise ValueError(
+                    f"Expected {self.n_features_in_} features in weights but got {len(self.weights)}"
+                )
+
+        # Set the reference
+        if self.reference is not None:
+            self.reference_ = np.array(self.reference)
+            self.indices_ = self._calculate_indices(self.reference_)
+            self.A_ = self._calculate_A(self.indices_, self.reference_)
+            self.weights_ = np.array(self.weights)
+            return self
+
+        if self.use_median:
+            self.reference_ = np.median(X, axis=0)
+            self.indices_ = self._calculate_indices(X[0])
+            self.A_ = self._calculate_A(self.indices_, self.reference_)
+            self.weights_ = np.array(self.weights)
+            return self
+
+        if self.use_mean:
+            self.reference_ = X.mean(axis=0)
+            self.indices_ = self._calculate_indices(X[0])
+            self.A_ = self._calculate_A(self.indices_, self.reference_)
+            self.weights_ = np.array(self.weights)
+            return self
+
+        raise ValueError("No reference was provided")
+
+    def transform(self, X: np.ndarray, y=None) -> np.ndarray:
+        """
+        Transform the input data by applying the multiplicative scatter
+        correction.
+
+        Parameters
+        ----------
+        X : np.ndarray of shape (n_samples, n_features)
+            The input data to transform.
+
+        y : None
+            Ignored.
+
+        Returns
+        -------
+        X_ : np.ndarray of shape (n_samples, n_features)
+            The transformed data.
+        """
+        # Check that the estimator is fitted
+        check_is_fitted(self, "n_features_in_")
+
+        # Check that X is a 2D array and has only finite values
+        X_ = validate_data(
+            self,
+            X,
+            y="no_validation",
+            ensure_2d=True,
+            copy=True,
+            reset=False,
+            dtype=np.float64,
+        )
+
+        if self.weights is None:
+            for i, x in enumerate(X_):
+                X_[i] = self._calculate_emsc(x)
+            return X_.reshape(-1, 1) if X_.ndim == 1 else X_
+
+        if self.weights is not None:
+            for i, x in enumerate(X_):
+                X_[i] = self._calculate_weighted_emsc(x)
+            return X_.reshape(-1, 1) if X_.ndim == 1 else X_
+
+    def _calculate_weighted_emsc(self, x):
+        reg = np.linalg.lstsq(
+            np.diag(self.weights_) @ self.A_, x * self.weights_, rcond=None
+        )[0]
+        x_ = (x - np.dot(self.A_[:, 0:-1], reg[0:-1])) / reg[-1]
+        return x_
+
+    def _calculate_emsc(self, x):
+        reg = np.linalg.lstsq(self.A_, x, rcond=None)[0]
+        x_ = (x - np.dot(self.A_[:, 0:-1], reg[0:-1])) / reg[-1]
+        return x_
+
+    def _calculate_indices(self, reference):
+        return np.linspace(0, len(reference) - 1, len(reference))
+
+    def _calculate_A(self, indices, reference):
+        return np.vstack(
+            [[np.power(indices, o) for o in range(self.order + 1)], reference]
+        ).T

chemotools/scatter/_multiplicative_scatter_correction.py

@@ -0,0 +1,169 @@
+from typing import Optional
+
+import numpy as np
+from sklearn.base import BaseEstimator, TransformerMixin, OneToOneFeatureMixin
+from sklearn.utils.validation import check_is_fitted, validate_data
+
+
+class MultiplicativeScatterCorrection(
+    TransformerMixin, OneToOneFeatureMixin, BaseEstimator
+):
+    """Multiplicative scatter correction (MSC) is a preprocessing technique for
+    removing scatter effects from spectra. It is based on fitting a linear
+    regression model to the spectrum using a reference spectrum. The reference
+    spectrum is usually a mean or median spectrum of a set of spectra.
+
+    Parameters
+    ----------
+    reference : np.ndarray of shape (n_freatures), optional
+        The reference spectrum to use for the correction. If None, the mean
+        spectrum will be used. The default is None.
+    use_mean : bool, optional
+        Whether to use the mean spectrum as the reference. The default is True.
+    use_median : bool, optional
+        Whether to use the median spectrum as the reference. The default is False.
+
+    Attributes
+    ----------
+    reference_ : np.ndarray
+        The reference spectrum used for the correction.
+    n_features_in_ : int
+        The number of features in the training data.
+
+    Raises
+    ------
+    ValueError
+        If no reference is provided.
+
+    """
+
+    def __init__(
+        self,
+        reference: Optional[np.ndarray] = None,
+        use_mean: bool = True,
+        use_median: bool = False,
+        weights: Optional[np.ndarray] = None,
+    ):
+        self.reference = reference
+        self.use_mean = use_mean
+        self.use_median = use_median
+        self.weights = weights
+
+    def fit(self, X: np.ndarray, y=None) -> "MultiplicativeScatterCorrection":
+        """
+        Fit the transformer to the input data. If no reference is provided, the
+        mean or median spectrum will be calculated from the input data.
+
+        Parameters
+        ----------
+        X : np.ndarray of shape (n_samples, n_features)
+            The input data to fit the transformer to.
+
+        y : None
+            Ignored.
+
+        Returns
+        -------
+        self : MultiplicativeScatterCorrection
+            The fitted transformer.
+        """
+        # Check that X is a 2D array and has only finite values
+        X = validate_data(
+            self, X, y="no_validation", ensure_2d=True, reset=True, dtype=np.float64
+        )
+        # Check that the length of the reference is the same as the number of features
+        if self.reference is not None:
+            if len(self.reference) != self.n_features_in_:
+                raise ValueError(
+                    f"Expected {self.n_features_in_} features in reference but got {len(self.reference)}"
+                )
+
+        if self.weights is not None:
+            if len(self.weights) != self.n_features_in_:
+                raise ValueError(
+                    f"Expected {self.n_features_in_} features in weights but got {len(self.weights)}"
+                )
+
+        # Set the reference
+        if self.reference is not None:
+            self.reference_ = np.array(self.reference)
+            self.A_ = self._calculate_A(self.reference_)
+            self.weights_ = np.array(self.weights)
+            return self
+
+        if self.use_median:
+            self.reference_ = np.median(X, axis=0)
+            self.A_ = self._calculate_A(self.reference_)
+            self.weights_ = np.array(self.weights)
+            return self
+
+        if self.use_mean:
+            self.reference_ = X.mean(axis=0)
+            self.A_ = self._calculate_A(self.reference_)
+            self.weights_ = np.array(self.weights)
+            return self
+
+        raise ValueError("No reference was provided")
+
+    def transform(self, X: np.ndarray, y=None) -> np.ndarray:
+        """
+        Transform the input data by applying the multiplicative scatter
+        correction.
+
+        Parameters
+        ----------
+        X : np.ndarray of shape (n_samples, n_features)
+            The input data to transform.
+
+        y : None
+            Ignored.
+
+        Returns
+        -------
+        X_ : np.ndarray of shape (n_samples, n_features)
+            The transformed data.
+        """
+        # Check that the estimator is fitted
+        check_is_fitted(self, "n_features_in_")
+
+        # Check that X is a 2D array and has only finite values
+        X_ = validate_data(
+            self,
+            X,
+            y="no_validation",
+            ensure_2d=True,
+            copy=True,
+            reset=False,
+            dtype=np.float64,
+        )
+
+        # Check that the number of features is the same as the fitted data
+        if X_.shape[1] != self.n_features_in_:
+            raise ValueError(
+                f"Expected {self.n_features_in_} features but got {X_.shape[1]}"
+            )
+
+        # Calculate the multiplicative signal correction
+        if self.weights is None:
+            for i, x in enumerate(X_):
+                X_[i] = self._calculate_multiplicative_correction(x)
+            return X_.reshape(-1, 1) if X_.ndim == 1 else X_
+
+        if self.weights is not None:
+            for i, x in enumerate(X_):
+                X_[i] = self._calculate_weighted_multiplicative_correction(x)
+            return X_.reshape(-1, 1) if X_.ndim == 1 else X_
+
+    def _calculate_weighted_multiplicative_correction(self, x) -> np.ndarray:
+        m, c = np.linalg.lstsq(
+            np.diag(self.weights_) @ self.A_, x * self.weights_, rcond=None
+        )[0]
+        return (x - c) / m
+
+    def _calculate_multiplicative_correction(self, x) -> np.ndarray:
+        m, c = np.linalg.lstsq(self.A_, x, rcond=None)[0]
+        return (x - c) / m
+
+    def _calculate_A(self, reference):
+        ones = np.ones(reference.shape[0])
+        return np.vstack([reference, ones]).T

chemotools/scatter/_robust_normal_variate.py

@@ -0,0 +1,101 @@
+import numpy as np
+from sklearn.base import BaseEstimator, TransformerMixin, OneToOneFeatureMixin
+from sklearn.utils.validation import check_is_fitted, validate_data
+
+
+class RobustNormalVariate(TransformerMixin, OneToOneFeatureMixin, BaseEstimator):
+    """
+    A transformer that calculates the robust normal variate of the input data.
+
+    Parameters
+    ----------
+    percentile : float, optional
+        The percentile to use for the robust normal variate. The value should be
+        between 0 and 100. The default is 25.
+
+    Methods
+    -------
+    fit(X, y=None)
+        Fit the transformer to the input data.
+
+    transform(X, y=0, copy=True)
+        Transform the input data by calculating the standard normal variate.
+
+    References
+    ----------
+    Q. Guo, W. Wu, D.L. Massart. The robust normal variate transform for pattern
+    recognition with near-infrared data. doi:10.1016/S0003-2670(98)00737-5
+    """
+
+    def __init__(self, percentile: float = 25):
+        self.percentile = percentile
+
+    def fit(self, X: np.ndarray, y=None) -> "RobustNormalVariate":
+        """
+        Fit the transformer to the input data.
+
+        Parameters
+        ----------
+        X : np.ndarray of shape (n_samples, n_features)
+            The input data to fit the transformer to.
+
+        y : None
+            Ignored.
+
+        Returns
+        -------
+        self : RobustNormalVariate
+            The fitted transformer.
+        """
+        # Check that X is a 2D array and has only finite values
+        X = validate_data(
+            self, X, y="no_validation", ensure_2d=True, reset=True, dtype=np.float64
+        )
+        return self
+
+    def transform(self, X: np.ndarray, y=None) -> np.ndarray:
+        """
+        Transform the input data by calculating the standard normal variate.
+
+        Parameters
+        ----------
+        X : np.ndarray of shape (n_samples, n_features)
+            The input data to transform.
+
+        y : None
+            Ignored.
+
+        Returns
+        -------
+        X_ : np.ndarray of shape (n_samples, n_features)
+            The transformed data.
+        """
+        # Check that the estimator is fitted
+        check_is_fitted(self, "n_features_in_")
+
+        # Check that X is a 2D array and has only finite values
+        X_ = validate_data(
+            self,
+            X,
+            y="no_validation",
+            ensure_2d=True,
+            copy=True,
+            reset=False,
+            dtype=np.float64,
+        )
+
+        # Check that the number of features is the same as the fitted data
+        if X_.shape[1] != self.n_features_in_:
+            raise ValueError(
+                f"Expected {self.n_features_in_} features but got {X_.shape[1]}"
+            )
+
+        # Calculate the standard normal variate
+        for i, x in enumerate(X_):
+            X_[i] = self._calculate_robust_normal_variate(x)
+
+        return X_.reshape(-1, 1) if X_.ndim == 1 else X_
+
+    def _calculate_robust_normal_variate(self, x) -> np.ndarray:
+        percentile = np.percentile(x, self.percentile)
+        return (x - percentile) / np.std(x[x <= percentile])

chemotools/scatter/{standard_normal_variate.py → _standard_normal_variate.py}

@@ -1,22 +1,12 @@
 import numpy as np
 from sklearn.base import BaseEstimator, TransformerMixin, OneToOneFeatureMixin
-from sklearn.utils.validation import check_is_fitted
+from sklearn.utils.validation import check_is_fitted, validate_data
 
-from chemotools.utils.check_inputs import check_input
 
-
-class StandardNormalVariate(OneToOneFeatureMixin, BaseEstimator, TransformerMixin):
+class StandardNormalVariate(TransformerMixin, OneToOneFeatureMixin, BaseEstimator):
     """
     A transformer that calculates the standard normal variate of the input data.
 
-    Attributes
-    ----------
-    n_features_in_ : int
-        The number of features in the input data.
-
-    _is_fitted : bool
-        Whether the transformer has been fitted to data.
-
     Methods
     -------
     fit(X, y=None)
@@ -25,10 +15,11 @@ class StandardNormalVariate(OneToOneFeatureMixin, BaseEstimator, TransformerMixi
     transform(X, y=0, copy=True)
         Transform the input data by calculating the standard normal variate.
     """
+
     def fit(self, X: np.ndarray, y=None) -> "StandardNormalVariate":
         """
         Fit the transformer to the input data.
-        
+
         Parameters
         ----------
         X : np.ndarray of shape (n_samples, n_features)
@@ -43,14 +34,9 @@ class StandardNormalVariate(OneToOneFeatureMixin, BaseEstimator, TransformerMixi
             The fitted transformer.
         """
         # Check that X is a 2D array and has only finite values
-        X = check_input(X)
-
-        # Set the number of features
-        self.n_features_in_ = X.shape[1]
-
-        # Set the fitted attribute to True
-        self._is_fitted = True
-
+        X = validate_data(
+            self, X, y="no_validation", ensure_2d=True, reset=True, dtype=np.float64
+        )
         return self
 
     def transform(self, X: np.ndarray, y=None) -> np.ndarray:
@@ -71,15 +57,24 @@ class StandardNormalVariate(OneToOneFeatureMixin, BaseEstimator, TransformerMixi
             The transformed data.
         """
         # Check that the estimator is fitted
-        check_is_fitted(self, "_is_fitted")
+        check_is_fitted(self, "n_features_in_")
 
         # Check that X is a 2D array and has only finite values
-        X = check_input(X)
-        X_ = X.copy()
+        X_ = validate_data(
+            self,
+            X,
+            y="no_validation",
+            ensure_2d=True,
+            copy=True,
+            reset=False,
+            dtype=np.float64,
+        )
 
         # Check that the number of features is the same as the fitted data
         if X_.shape[1] != self.n_features_in_:
-            raise ValueError(f"Expected {self.n_features_in_} features but got {X_.shape[1]}")
+            raise ValueError(
+                f"Expected {self.n_features_in_} features but got {X_.shape[1]}"
+            )
 
         # Calculate the standard normal variate
         for i, x in enumerate(X_):
@@ -88,4 +83,4 @@ class StandardNormalVariate(OneToOneFeatureMixin, BaseEstimator, TransformerMixi
         return X_.reshape(-1, 1) if X_.ndim == 1 else X_
 
     def _calculate_standard_normal_variate(self, x) -> np.ndarray:
-        return (x - x.mean()) / x.std()
+        return (x - x.mean()) / x.std()

chemotools/smooth/__init__.py

@@ -1,4 +1,6 @@
-from .mean_filter import MeanFilter
-from .median_filter import MedianFilter
-from .savitzky_golay_filter import SavitzkyGolayFilter
-from .whittaker_smooth import WhittakerSmooth
+from ._mean_filter import MeanFilter
+from ._median_filter import MedianFilter
+from ._savitzky_golay_filter import SavitzkyGolayFilter
+from ._whittaker_smooth import WhittakerSmooth
+
+__all__ = ["MeanFilter", "MedianFilter", "SavitzkyGolayFilter", "WhittakerSmooth"]

chemotools/smooth/{mean_filter.py → _mean_filter.py}

@@ -1,12 +1,10 @@
 import numpy as np
 from scipy.ndimage import uniform_filter1d
 from sklearn.base import BaseEstimator, TransformerMixin, OneToOneFeatureMixin
-from sklearn.utils.validation import check_is_fitted
+from sklearn.utils.validation import check_is_fitted, validate_data
 
-from chemotools.utils.check_inputs import check_input
 
-
-class MeanFilter(OneToOneFeatureMixin, BaseEstimator, TransformerMixin):
+class MeanFilter(TransformerMixin, OneToOneFeatureMixin, BaseEstimator):
     """
     A transformer that calculates the mean filter of the input data.
 
@@ -14,19 +12,11 @@ class MeanFilter(OneToOneFeatureMixin, BaseEstimator, TransformerMixin):
     ----------
     window_size : int, optional
         The size of the window to use for the mean filter. Must be odd. Default is 3.
-    
+
     mode : str, optional
         The mode to use for the mean filter. Can be "nearest", "constant", "reflect",
         "wrap", "mirror" or "interp". Default is "nearest".
 
-    Attributes
-    ----------
-    n_features_in_ : int
-        The number of features in the input data.
-
-    _is_fitted : bool
-        Whether the transformer has been fitted to data.
-
     Methods
     -------
     fit(X, y=None)
@@ -35,7 +25,8 @@ class MeanFilter(OneToOneFeatureMixin, BaseEstimator, TransformerMixin):
     transform(X, y=0, copy=True)
         Transform the input data by calculating the mean filter.
     """
-    def __init__(self, window_size: int = 3, mode='nearest') -> None:
+
+    def __init__(self, window_size: int = 3, mode="nearest") -> None:
         self.window_size = window_size
         self.mode = mode
 
@@ -57,14 +48,9 @@ class MeanFilter(OneToOneFeatureMixin, BaseEstimator, TransformerMixin):
             The fitted transformer.
         """
         # Check that X is a 2D array and has only finite values
-        X = check_input(X)
-
-        # Set the number of features
-        self.n_features_in_ = X.shape[1]
-
-        # Set the fitted attribute to True
-        self._is_fitted = True
-
+        X = validate_data(
+            self, X, y="no_validation", ensure_2d=True, reset=True, dtype=np.float64
+        )
         return self
 
     def transform(self, X: np.ndarray, y=None) -> np.ndarray:
@@ -85,11 +71,18 @@ class MeanFilter(OneToOneFeatureMixin, BaseEstimator, TransformerMixin):
             The transformed data.
         """
         # Check that the estimator is fitted
-        check_is_fitted(self, "_is_fitted")
+        check_is_fitted(self, "n_features_in_")
 
         # Check that X is a 2D array and has only finite values
-        X = check_input(X)
-        X_ = X.copy()
+        X_ = validate_data(
+            self,
+            X,
+            y="no_validation",
+            ensure_2d=True,
+            copy=True,
+            reset=False,
+            dtype=np.float64,
+        )
 
         if X_.shape[1] != self.n_features_in_:
             raise ValueError(