chemotools 0.1.9__py3-none-any.whl → 0.1.11__py3-none-any.whl

chemotools/outliers/q_residuals.py

@@ -0,0 +1,228 @@
+from typing import Optional, Literal, Union
+
+import numpy as np
+
+from scipy.stats import norm, chi2
+from sklearn.pipeline import Pipeline
+from sklearn.utils.validation import validate_data, check_is_fitted
+
+from ._base import _ModelResidualsBase, ModelTypes
+from .utils import calculate_residual_spectrum
+
+
+class QResiduals(_ModelResidualsBase):
+    """
+    Calculate Q residuals (Squared Prediction Error - SPE) for PCA or PLS models.
+
+    Parameters
+    ----------
+    model : Union[ModelType, Pipeline]
+        A fitted PCA/PLS model or Pipeline ending with such a model.
+
+    confidence : float, default=0.95
+        Confidence level for statistical calculations (between 0 and 1).
+
+    method : str, default="jackson-mudholkar"
+        The method used to compute the confidence threshold for Q residuals.
+        Options:
+        - "chi-square" : Uses mean and standard deviation to approximate Q residuals threshold.
+        - "jackson-mudholkar" : Uses eigenvalue-based analytical approximation.
+        - "percentile" : Uses empirical percentile threshold.
+
+    Attributes
+    ----------
+    estimator_ : ModelType
+        The fitted model of type _BasePCA or _PLS.
+
+    transformer_ : Optional[Pipeline]
+        Preprocessing steps before the model.
+
+    n_features_in_ : int
+        Number of features in the input data.
+
+    n_components_ : int
+        Number of components in the model.
+
+    n_samples_ : int
+        Number of samples used to train the model.
+
+    critical_value_ : float
+        The calculated critical value for outlier detection.
+
+    References
+    ----------
+    Johan A. Westerhuis, Stephen P. Gurden, Age K. Smilde (2001) Generalized contribution plots in multivariate statistical process
+    monitoring  Chemometrics and Intelligent Laboratory Systems 51 2000 95–114
+    """
+
+    def __init__(
+        self,
+        model: Union[ModelTypes, Pipeline],
+        confidence: float = 0.95,
+        method: Literal[
+            "chi-square", "jackson-mudholkar", "percentile"
+        ] = "jackson-mudholkar",
+    ) -> None:
+        self.model, self.confidence, self.method = model, confidence, method
+        super().__init__(model, confidence)
+
+    def fit(self, X: np.ndarray, y: Optional[np.ndarray] = None) -> "QResiduals":
+        """
+        Fit the Q Residuals model by computing residuals from the training set.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Training data.
+
+        Returns
+        -------
+        self : object
+            Fitted instance of QResiduals.
+        """
+        X = validate_data(self, X, ensure_2d=True, dtype=np.float64)
+
+        if self.transformer_:
+            X = self.transformer_.fit_transform(X)
+
+        # Compute the critical threshold using the chosen method
+        self.critical_value_ = self._calculate_critical_value(X)
+
+        return self
+
+    def predict(self, X: np.ndarray) -> np.ndarray:
+        """Identify outliers in the input data based on Q residuals threshold.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Input data.
+
+        Returns
+        -------
+        ndarray of shape (n_samples,)
+            Boolean array indicating outliers (-1 for outliers, 1 for normal data).
+        """
+        # Check the estimator has been fitted
+        check_is_fitted(self, ["critical_value_"])
+
+        # Validate the input data
+        X = validate_data(
+            self, X, y="no_validation", ensure_2d=True, reset=True, dtype=np.float64
+        )
+
+        # Calculate outliers based on the Q residuals
+        Q_residuals = self.predict_residuals(X, validate=False)
+        return np.where(Q_residuals > self.critical_value_, -1, 1)
+
+    def predict_residuals(
+        self, X: np.ndarray, y: Optional[np.ndarray] = None, validate: bool = True
+    ) -> np.ndarray:
+        """Calculate Q residuals (Squared Prediction Error - SPE) for input data.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Input data.
+
+        validate : bool, default=True
+            Whether to validate the input data.
+
+        Returns
+        -------
+        ndarray of shape (n_samples,)
+            Q residuals for each sample.
+        """
+        # Check the estimator has been fitted
+        check_is_fitted(self, ["critical_value_"])
+
+        # Validate the input data
+        if validate:
+            X = validate_data(self, X, ensure_2d=True, dtype=np.float64)
+
+        # Apply preprocessing if available
+        if self.transformer_:
+            X = self.transformer_.transform(X)
+
+        # Compute reconstruction error (Q residuals)
+        residual = calculate_residual_spectrum(X, self.estimator_)
+        Q_residuals = np.sum(residual**2, axis=1)
+
+        return Q_residuals
+
+    def _calculate_critical_value(
+        self,
+        X: np.ndarray,
+    ) -> float:
+        """Calculate the critical value for outlier detection.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Input data.
+
+        X_reconstructed : array-like of shape (n_samples, n_features)
+            Reconstructed input data.
+
+        method : str Literal["chi-square", "jackson-mudholkar", "percentile"]
+            The method used to compute the confidence threshold for Q residuals.
+
+        Returns
+        -------
+        float
+            The calculated critical value for outlier detection.
+
+        """
+        # Compute Q residuals for training data
+        residuals = calculate_residual_spectrum(X, self.estimator_)
+
+        if self.method == "chi-square":
+            return self._chi_square_threshold(residuals)
+
+        elif self.method == "jackson-mudholkar":
+            return self._jackson_mudholkar_threshold(residuals)
+
+        elif self.method == "percentile":
+            Q_residuals = np.sum((residuals) ** 2, axis=1)
+            return self._percentile_threshold(Q_residuals)
+
+        else:
+            raise ValueError(
+                "Invalid method. Choose from 'chi-square', 'jackson-mudholkar', or 'percentile'."
+            )
+
+    def _chi_square_threshold(self, residuals: np.ndarray) -> float:
+        """Compute Q residual threshold using Chi-Square Approximation."""
+        eigenvalues = np.linalg.trace(np.cov(residuals.T))
+
+        theta_1 = np.sum(eigenvalues)
+        theta_2 = np.sum(eigenvalues**2)
+        # Degrees of freedom approximation
+        g = theta_2 / theta_1
+        h = (2 * theta_1**2) / theta_2
+
+        # Compute chi-square critical value at given confidence level
+        chi_critical = chi2.ppf(self.confidence, df=h)
+
+        # Compute final Q residual threshold
+        return g * chi_critical
+
+    def _jackson_mudholkar_threshold(self, residuals: np.ndarray) -> float:
+        """Compute Q residual threshold using Jackson & Mudholkar’s analytical method."""
+
+        eigenvalues = np.linalg.trace(np.cov(residuals.T))
+        theta_1 = np.sum(eigenvalues)
+        theta_2 = np.sum(eigenvalues**2)
+        theta_3 = np.sum(eigenvalues**3)
+        z_alpha = norm.ppf(self.confidence)
+
+        h0 = 1 - (2 * theta_1 * theta_3) / (3 * theta_2**2)
+
+        term1 = theta_2 * h0 * (1 - h0) / theta_1**2
+        term2 = np.sqrt(z_alpha * 2 * theta_2 * h0**2) / theta_1
+
+        return theta_1 * (1 - term1 + term2) ** (1 / h0)
+
+    def _percentile_threshold(self, Q_residuals: np.ndarray) -> float:
+        """Compute Q residual threshold using the empirical percentile method."""
+        return np.percentile(Q_residuals, self.confidence * 100)

chemotools/outliers/studentized_residuals.py

@@ -0,0 +1,198 @@
+from typing import Optional, Union
+import numpy as np
+
+from sklearn.cross_decomposition._pls import _PLS
+from sklearn.pipeline import Pipeline
+from sklearn.utils.validation import validate_data, check_is_fitted
+
+
+from ._base import _ModelResidualsBase, ModelTypes
+from .leverage import calculate_leverage
+
+
+class StudentizedResiduals(_ModelResidualsBase):
+    """
+    Calculate the Studentized Residuals on a _PLS model preditions.
+
+    Parameters
+    ----------
+    model : Union[ModelType, Pipeline]
+        A fitted _PLS model or Pipeline ending with such a model
+
+    Attributes
+    ----------
+    estimator_ : ModelType
+        The fitted model of type _BasePCA or _PLS
+
+    transformer_ : Optional[Pipeline]
+        Preprocessing steps before the model
+
+    References
+    ----------
+
+    """
+
+    def __init__(self, model: Union[_PLS, Pipeline], confidence=0.95) -> None:
+        self.model, self.confidence = model, confidence
+        super().__init__(model, confidence)
+
+    def fit(self, X: np.ndarray, y: Optional[np.ndarray]) -> "StudentizedResiduals":
+        """
+        Fit the model to the input data.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Input data
+
+        y : array-like of shape (n_samples,)
+            Target data
+        """
+        # Validate the input data
+        X = validate_data(
+            self, X, y="no_validation", ensure_2d=True, reset=True, dtype=np.float64
+        )
+
+        # Preprocess the data
+        if self.transformer_:
+            X = self.transformer_.transform(X)
+
+        # Calculate y residuals
+        y_residuals = y - self.estimator_.predict(X)
+        y_residuals = (
+            y_residuals.reshape(-1, 1) if len(y_residuals.shape) == 1 else y_residuals
+        )
+
+        # Calculate the studentized residuals
+        studentized_residuals = calculate_studentized_residuals(
+            self.estimator_, X, y_residuals
+        )
+
+        # Calculate the critical threshold
+        self.critical_value_ = self._calculate_critical_value(studentized_residuals)
+
+        return self
+
+    def predict(self, X: np.ndarray, y: Optional[np.ndarray]) -> np.ndarray:
+        """Calculate studentized residuals in the model predictions. and return a boolean array indicating outliers.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Input data
+
+        y : array-like of shape (n_samples,)
+            Target data
+
+        Returns
+        -------
+        ndarray of shape (n_samples,)
+            Studentized residuals of the predictions
+        """
+        # Check the estimator has been fitted
+        check_is_fitted(self, ["critical_value_"])
+
+        # Validate the input data
+        X = validate_data(
+            self, X, y="no_validation", ensure_2d=True, reset=True, dtype=np.float64
+        )
+
+        # Preprocess the data
+        if self.transformer_:
+            X = self.transformer_.transform(X)
+
+        # Calculate y residuals
+        y_residuals = y - self.estimator_.predict(X)
+        y_residuals = (
+            y_residuals.reshape(-1, 1) if len(y_residuals.shape) == 1 else y_residuals
+        )
+
+        # Calculate the studentized residuals
+        studentized_residuals = calculate_studentized_residuals(
+            self.estimator_, X, y_residuals
+        )
+        return np.where(studentized_residuals > self.critical_value_, -1, 1)
+
+    def predict_residuals(
+        self, X: np.ndarray, y: Optional[np.ndarray], validate: bool = True
+    ) -> np.ndarray:
+        """Calculate the studentized residuals of the model predictions.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Input data
+
+        y : array-like of shape (n_samples,)
+            Target values
+
+        Returns
+        -------
+        ndarray of shape (n_samples,)
+            Studentized residuals of the model predictions
+        """
+        # Check the estimator has been fitted
+        check_is_fitted(self, ["critical_value_"])
+
+        # Validate the input data
+        if validate:
+            X = validate_data(self, X, ensure_2d=True, dtype=np.float64)
+
+        # Apply preprocessing if available
+        if self.transformer_:
+            X = self.transformer_.transform(X)
+
+        # Calculate y residuals
+        y_residuals = y - self.estimator_.predict(X)
+        y_residuals = (
+            y_residuals.reshape(-1, 1) if len(y_residuals.shape) == 1 else y_residuals
+        )
+
+        return calculate_studentized_residuals(self.estimator_, X, y_residuals)
+
+    def _calculate_critical_value(self, X: np.ndarray) -> float:
+        """Calculate the critical value for outlier detection.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples,)
+            Studentized residuals
+
+        Returns
+        -------
+        float
+            The calculated critical value for outlier detection
+        """
+
+        return np.percentile(X, self.confidence * 100) if X is not None else 0.0
+
+
+def calculate_studentized_residuals(
+    model: ModelTypes, X: np.ndarray, y_residuals: np.ndarray
+) -> np.ndarray:
+    """Calculate the studentized residuals of the model predictions.
+
+    Parameters
+    ----------
+    model : ModelTypes
+        A fitted model
+
+    X : array-like of shape (n_samples, n_features)
+        Input data
+
+    y : array-like of shape (n_samples,)
+        Target values
+
+    Returns
+    -------
+    ndarray of shape (n_samples,)
+        Studentized residuals of the model predictions
+    """
+
+    # Calculate the leverage of the samples
+    leverage = calculate_leverage(X, model)
+
+    # Calculate the standard deviation of the residuals
+    std = np.sqrt(np.sum(y_residuals**2, axis=0) / (X.shape[0] - model.n_components))
+
+    return (y_residuals / (std * np.sqrt(1 - leverage.reshape(-1, 1)))).flatten()

chemotools/outliers/utils.py

@@ -0,0 +1,51 @@
+import numpy as np
+
+from ._base import ModelTypes
+
+
+def calculate_decoded_spectrum(X: np.ndarray, estimator: ModelTypes):
+    """
+    Calculate the decoded spectrum for a given transformed (preprocessed!!) spectrum and estimator from the latent space.
+
+    Parameters
+    ----------
+    spectrum : np.ndarray
+        The transformed spectrum data.
+
+    estimator : ModelTypes
+        The fitted PCA or PLS model.
+
+    Returns
+    -------
+    np.ndarray
+        The decoded spectrum.
+    """
+    # Project the transformed spectrum onto the latent space
+    X_transformed = estimator.transform(X)
+
+    # Decode the spectrum back to the original space
+    return estimator.inverse_transform(X_transformed)
+
+
+def calculate_residual_spectrum(X: np.ndarray, estimator: ModelTypes):
+    """
+    Calculate the residual spectrum for a given transformed (preprocessed!!) spectrum and estimator.
+
+    Parameters
+    ----------
+    spectrum : np.ndarray
+        The transformed spectrum data.
+
+    estimator : ModelTypes
+        The fitted PCA or PLS model.
+
+    Returns
+    -------
+    np.ndarray
+        The residual spectrum.
+    """
+    # Compute the reconstruction error (Q residuals)
+    decoded_spectrum = calculate_decoded_spectrum(X, estimator)
+
+    # Calculate the residual
+    return X - decoded_spectrum

{chemotools-0.1.9.dist-info → chemotools-0.1.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chemotools
-Version: 0.1.9
+Version: 0.1.11
 Summary: chemotools: A Python Package that Integrates Chemometrics and scikit-learn
 License: MIT
 Author: Pau Cabaneros
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: numpy (>=2.0.0,<3.0.0)
 Requires-Dist: pandas (>=2.0.0,<3.0.0)
 Requires-Dist: polars (>=1.17.0,<2.0.0)
-Requires-Dist: pyarrow (>=18.0.0,<19.0.0)
+Requires-Dist: pyarrow (>=18,<21)
 Requires-Dist: scikit-learn (>=1.4.0,<2.0.0)
 Description-Content-Type: text/markdown
 

{chemotools-0.1.9.dist-info → chemotools-0.1.11.dist-info}/RECORD

@@ -1,8 +1,9 @@
 chemotools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-chemotools/augmentation/__init__.py,sha256=iRltqvskLJAhwxxlTtGPWJfR5XiwvQfqbv4QdUMF9BU,318
+chemotools/augmentation/__init__.py,sha256=ohlRHgRWTkvNpO3RikKtowzboqunQqx0WqtNccuWOHw,397
 chemotools/augmentation/_add_noise.py,sha256=fkTJfIYtZXezcjy6Vz8asIhpBoVp4oaIifppK9vZpM8,4362
 chemotools/augmentation/_baseline_shift.py,sha256=kIlYvmKS9pu9vh_-eZ7PSHPuH_58V9mgYbSJt6Gq3BA,3476
 chemotools/augmentation/_fractional_shift.py,sha256=dJ0Vuc-U02HhjKkOwc48qnOksZYgbHwL2ko7tWCZTQU,6916
+chemotools/augmentation/_gaussian_broadening.py,sha256=dJsPlTKqpecKaCDU3vOvedIb-t_HyCkQprxNv0DmYZQ,4236
 chemotools/augmentation/_index_shift.py,sha256=BTtadweDvvMtiF8t7ldwsE6Kl6FmKLCkVJjSzSWyIDs,6904
 chemotools/augmentation/_spectrum_scale.py,sha256=hMsmzXpssbI7tGm_YnQn9wjbByso3CgVxd3Hs8kfLS8,3442
 chemotools/baseline/__init__.py,sha256=VzoblGg8Hx_FkTc_n7a-ZjGvtKP8JE_NwJKWenGFQkM,584
@@ -26,9 +27,20 @@ chemotools/datasets/data/train_spectra.csv,sha256=iVF19W52NHlbqq8BbLomn8n47kSPT0
 chemotools/derivative/__init__.py,sha256=FkckdzO30jrRWPGpIU3cfnaTtxPtNT5Tb2G9F9PmVTw,134
 chemotools/derivative/_norris_william.py,sha256=rMY_yntpiB5fbSM1tPph4AaGmF1k-HqJp7o48ijePBs,4958
 chemotools/derivative/_savitzky_golay.py,sha256=CuCrKoLmrB1YmJ4ihIykgkL3tO3frqkStMogtsVhO3A,3632
-chemotools/feature_selection/__init__.py,sha256=1_i28hIxijjwhMypTy1w2fLbzXXVkKD5IYzzY8ZSuHw,117
+chemotools/feature_selection/__init__.py,sha256=e_GFVawlDNEQv3EqrGSXUr5cvDN1jckoxe2C2jRwVl8,222
+chemotools/feature_selection/_base.py,sha256=SIH6kl9AePVWTByL0OvJFfc2j3idqs7lm_7Zi1YMp4Y,2311
 chemotools/feature_selection/_index_selector.py,sha256=lNTP2b7P3doWl30KiAr3Xd2HOMxeUmj24MuqoXl4Voc,3556
 chemotools/feature_selection/_range_cut.py,sha256=lVVVC30ZsK2z9jsDGb_z6l8Ty2I89yM05_dIDbMP73Q,3564
+chemotools/feature_selection/_sr_selector.py,sha256=OaXkt3t_NvymgDy6R15ig87jhcb-vM7i63LgtsNdfZo,3969
+chemotools/feature_selection/_vip_selector.py,sha256=ZK3bhdpl3nBYt6xmuHq2IvWtpgJ8ZdElH06xnCFA-Xs,3835
+chemotools/outliers/__init__.py,sha256=wpdlyqU34n1Pb9kGCM4idhcok35WAakxEhzP0xeKaZw,272
+chemotools/outliers/_base.py,sha256=zl0LhRKjpvj5IbYc3su6zEZ7YZ0pDSR3yqNWt2qBjNA,5374
+chemotools/outliers/dmodx.py,sha256=sgizal_BDlqWTZNT8y2D_ImcKAJejXt6vqvFYk4Vqi0,5152
+chemotools/outliers/hotelling_t2.py,sha256=g_IOQD_rhKb3cjIJkn5OTto6bYClQtqXunG_02BSIs8,5087
+chemotools/outliers/leverage.py,sha256=hNQ_x68LPPTDZvSJP_eRqu3GoeV3OBU37VC_XTFEzvw,4250
+chemotools/outliers/q_residuals.py,sha256=sg7u8ockQvSSnXwNM4U-GITB-5OcbsDMX6Oig_TcONM,7598
+chemotools/outliers/studentized_residuals.py,sha256=1L-GiutuO1x9s3UKMOBpmhs2Q-UuDtfG2YLELIxiiao,5890
+chemotools/outliers/utils.py,sha256=SAjvtjl9oWHrQnkqGnDfYE4WWAgiL1RwnKmW-ql5TIc,1304
 chemotools/scale/__init__.py,sha256=eztqcHg-TKE1Rr0N9ArfytHk8teuqVfi4SZi2DS96vc,175
 chemotools/scale/_min_max_scaler.py,sha256=YvqRkV2pXu-viQrpjzWcp9KmSSCYSoubSnrZHRLqgKQ,3011
 chemotools/scale/_norm_scaler.py,sha256=CHWSir2q-pL1hxzw_ZB45yi4mw-SkJ4YOa1CUL4nm2I,2568
@@ -44,7 +56,7 @@ chemotools/smooth/_median_filter.py,sha256=9ndTJCwrZirWlvDNldiigMddy79KIGq9OwwYN
 chemotools/smooth/_savitzky_golay_filter.py,sha256=27iFUWxdL9_7oZabR0R5L0ZTpBmYfVUjx2XCTukihBE,3509
 chemotools/smooth/_whittaker_smooth.py,sha256=lpLAyf4GdyDW4ulT1nyEoK6xQEl2cVUKquawQdGWbHU,3571
 chemotools/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-chemotools-0.1.9.dist-info/LICENSE,sha256=qtyOy2wDQVX9hxp58h3T-6Lmfv-mSCHoSRkcLUdM9bg,1070
-chemotools-0.1.9.dist-info/METADATA,sha256=9sih25qSOTJX36gib96hxm-e86TorD040r_WNFzbc9U,5239
-chemotools-0.1.9.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-chemotools-0.1.9.dist-info/RECORD,,
+chemotools-0.1.11.dist-info/LICENSE,sha256=qtyOy2wDQVX9hxp58h3T-6Lmfv-mSCHoSRkcLUdM9bg,1070
+chemotools-0.1.11.dist-info/METADATA,sha256=Ne8xEa1cZUhbP-I4D1CFVvy8fhJANUjsY5cXRpNVV1k,5232
+chemotools-0.1.11.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+chemotools-0.1.11.dist-info/RECORD,,

{chemotools-0.1.9.dist-info → chemotools-0.1.11.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.0.1
+Generator: poetry-core 2.1.3
 Root-Is-Purelib: true
 Tag: py3-none-any