chemotools 0.0.22__py3-none-any.whl → 0.0.24__py3-none-any.whl

chemotools/scatter/extended_multiplicative_scatter_correction.py

@@ -1,33 +1,192 @@
 import numpy as np
-from sklearn.base import BaseEstimator, TransformerMixin
+from sklearn.base import BaseEstimator, TransformerMixin, OneToOneFeatureMixin
+from sklearn.preprocessing import StandardScaler
 from sklearn.utils.validation import check_is_fitted
 
 from chemotools.utils.check_inputs import check_input
 
-class ExtendedMultiplicativeScatterCorrection(BaseEstimator, TransformerMixin):
-    def __init__(self):
-        self.ref_spec = None
-        self.coeffs = None
-
-    def fit(self, X, ref_spec=None):
-        if ref_spec is None:
-            # Use mean spectrum as reference if none provided
-            ref_spec = np.mean(X, axis=0)
-        self.ref_spec = ref_spec
-
-        # Calculate the mean spectrum
-        mean_spec = np.mean(X, axis=0)
-
-        # Fit a linear model to the reference spectrum
-        coeffs = np.polyfit(mean_spec, ref_spec, deg=1)
-        self.coeffs = coeffs
-
-    def transform(self, X):
-        # Divide the spectra by the linear model
-        X_emsc = X / np.polyval(self.coeffs, X.mean(axis=1))
-        return X_emsc
-
-    def fit_transform(self, X, ref_spec=None):
-        self.fit(X, ref_spec=ref_spec)
-        X_emsc = self.transform(X)
-        return X_emsc
+
+class ExtendedMultiplicativeScatterCorrection(
+    OneToOneFeatureMixin, BaseEstimator, TransformerMixin
+):
+    """Extended multiplicative scatter correction (EMSC) is a preprocessing technique for
+    removing non linear scatter effects from spectra. It is based on fitting a polynomial
+    regression model to the spectrum using a reference spectrum. The reference spectrum
+    can be the mean or median spectrum of a set of spectra or a selected reerence.
+
+    Note that this implementation does not include further extensions of the model using
+    orthogonal subspace models.
+
+    Parameters
+    ----------
+    reference : np.ndarray, optional
+        The reference spectrum to use for the correction. If None, the mean
+        spectrum will be used. The default is None.
+    use_mean : bool, optional
+        Whether to use the mean spectrum as the reference. The default is True.
+    use_median : bool, optional
+        Whether to use the median spectrum as the reference. The default is False.
+    order : int, optional
+        The order of the polynomial to fit to the spectrum. The default is 2.
+    weights : np.ndarray, optional
+        The weights to use for the weighted EMSC. If None, the standard EMSC
+        will be used. The default is None.
+
+
+    Attributes
+    ----------
+    reference_ : np.ndarray
+        The reference spectrum used for the correction.
+    n_features_in_ : int
+        The number of features in the training data.
+
+    References
+    ----------
+    Nils Kristian Afseth, Achim Kohler. Extended multiplicative signal correction
+    in vibrational spectroscopy, a tutorial, doi:10.1016/j.chemolab.2012.03.004
+
+    Valeria Tafintseva et al. Correcting replicate variation in spectroscopic data by machine learning and
+    model-based pre-processing, doi:10.1016/j.chemolab.2021.104350
+    """
+
+    def __init__(
+        self,
+        reference: np.ndarray = None,
+        use_mean: bool = True,
+        use_median: bool = False,
+        order: int = 2,
+        weights: np.ndarray = None,
+    ):
+        self.reference = reference
+        self.use_mean = use_mean
+        self.use_median = use_median
+        self.order = order
+        self.weights = weights
+
+    def fit(self, X: np.ndarray, y=None) -> "ExtendedMultiplicativeScatterCorrection":
+        """
+        Fit the transformer to the input data. If no reference is provided, the
+        mean or median spectrum will be calculated from the input data.
+
+        Parameters
+        ----------
+        X : np.ndarray of shape (n_samples, n_features)
+            The input data to fit the transformer to.
+
+        y : None
+            Ignored.
+
+        Returns
+        -------
+        self : MultiplicativeScatterCorrection
+            The fitted transformer.
+        """
+        # Check that X is a 2D array and has only finite values
+        X = check_input(X)
+
+        # Set the number of features
+        self.n_features_in_ = X.shape[1]
+
+        # Set the fitted attribute to True
+        self._is_fitted = True
+
+        # Check that the length of the reference is the same as the number of features
+        if self.reference is not None:
+            if len(self.reference) != self.n_features_in_:
+                raise ValueError(
+                    f"Expected {self.n_features_in_} features in reference but got {len(self.reference)}"
+                )
+
+        if self.weights is not None:
+            if len(self.weights) != self.n_features_in_:
+                raise ValueError(
+                    f"Expected {self.n_features_in_} features in weights but got {len(self.weights)}"
+                )
+
+        # Set the reference
+        if self.reference is not None:
+            self.reference_ = np.array(self.reference)
+            self.indices_ = self._calculate_indices(self.reference_)
+            self.A_ = self._calculate_A(self.indices_, self.reference_)
+            self.weights_ = np.array(self.weights)
+            return self
+
+        if self.use_median:
+            self.reference_ = np.median(X, axis=0)
+            self.indices_ = self._calculate_indices(X[0])
+            self.A_ = self._calculate_A(self.indices_, self.reference_)
+            self.weights_ = np.array(self.weights)
+            return self
+
+        if self.use_mean:
+            self.reference_ = X.mean(axis=0)
+            self.indices_ = self._calculate_indices(X[0])
+            self.A_ = self._calculate_A(self.indices_, self.reference_)
+            self.weights_ = np.array(self.weights)
+            return self
+
+        raise ValueError("No reference was provided")
+
+    def transform(self, X: np.ndarray, y=None) -> np.ndarray:
+        """
+        Transform the input data by applying the multiplicative scatter
+        correction.
+
+        Parameters
+        ----------
+        X : np.ndarray of shape (n_samples, n_features)
+            The input data to transform.
+
+        y : None
+            Ignored.
+
+        Returns
+        -------
+        X_ : np.ndarray of shape (n_samples, n_features)
+            The transformed data.
+        """
+        # Check that the estimator is fitted
+        check_is_fitted(self, "_is_fitted")
+
+        # Check that X is a 2D array and has only finite values
+        X = check_input(X)
+        X_ = X.copy()
+
+        # Check that the number of features is the same as the fitted data
+        if X_.shape[1] != self.n_features_in_:
+            raise ValueError(
+                f"Expected {self.n_features_in_} features but got {X_.shape[1]}"
+            )
+
+        # Calculate the extended multiplicative scatter correction
+        X_ = X.copy()
+
+        if self.weights is None:
+            for i, x in enumerate(X_):
+                X_[i] = self._calculate_emsc(x)
+            return X_.reshape(-1, 1) if X_.ndim == 1 else X_
+
+        if self.weights is not None:
+            for i, x in enumerate(X_):
+                X_[i] = self._calculate_weighted_emsc(x)
+            return X_.reshape(-1, 1) if X_.ndim == 1 else X_
+
+    def _calculate_weighted_emsc(self, x):
+        reg = np.linalg.lstsq(
+            np.diag(self.weights_) @ self.A_, x * self.weights_, rcond=None
+        )[0]
+        x_ = (x - np.dot(self.A_[:, 0:-1], reg[0:-1])) / reg[-1]
+        return x_
+
+    def _calculate_emsc(self, x):
+        reg = np.linalg.lstsq(self.A_, x, rcond=None)[0]
+        x_ = (x - np.dot(self.A_[:, 0:-1], reg[0:-1])) / reg[-1]
+        return x_
+
+    def _calculate_indices(self, reference):
+        return np.linspace(0, len(reference) - 1, len(reference))
+
+    def _calculate_A(self, indices, reference):
+        return np.vstack(
+            [[np.power(indices, o) for o in range(self.order + 1)], reference]
+        ).T

chemotools/scatter/multiplicative_scatter_correction.py

@@ -5,7 +5,9 @@ from sklearn.utils.validation import check_is_fitted
 from chemotools.utils.check_inputs import check_input
 
 
-class MultiplicativeScatterCorrection(OneToOneFeatureMixin, BaseEstimator, TransformerMixin):
+class MultiplicativeScatterCorrection(
+    OneToOneFeatureMixin, BaseEstimator, TransformerMixin
+):
     """Multiplicative scatter correction (MSC) is a preprocessing technique for
     removing scatter effects from spectra. It is based on fitting a linear
     regression model to the spectrum using a reference spectrum. The reference
@@ -13,7 +15,7 @@ class MultiplicativeScatterCorrection(OneToOneFeatureMixin, BaseEstimator, Trans
 
     Parameters
     ----------
-    reference : np.ndarray, optional
+    reference : np.ndarray of shape (n_freatures), optional
         The reference spectrum to use for the correction. If None, the mean
         spectrum will be used. The default is None.
     use_mean : bool, optional
@@ -32,17 +34,20 @@ class MultiplicativeScatterCorrection(OneToOneFeatureMixin, BaseEstimator, Trans
     ------
     ValueError
         If no reference is provided.
-    
+
     """
+
     def __init__(
         self,
         reference: np.ndarray = None,
         use_mean: bool = True,
         use_median: bool = False,
+        weights: np.ndarray = None,
     ):
         self.reference = reference
         self.use_mean = use_mean
         self.use_median = use_median
+        self.weights = weights
 
     def fit(self, X: np.ndarray, y=None) -> "MultiplicativeScatterCorrection":
         """
@@ -71,17 +76,36 @@ class MultiplicativeScatterCorrection(OneToOneFeatureMixin, BaseEstimator, Trans
         # Set the fitted attribute to True
         self._is_fitted = True
 
+        # Check that the length of the reference is the same as the number of features
+        if self.reference is not None:
+            if len(self.reference) != self.n_features_in_:
+                raise ValueError(
+                    f"Expected {self.n_features_in_} features in reference but got {len(self.reference)}"
+                )
+
+        if self.weights is not None:
+            if len(self.weights) != self.n_features_in_:
+                raise ValueError(
+                    f"Expected {self.n_features_in_} features in weights but got {len(self.weights)}"
+                )
+
         # Set the reference
-        if self.reference is None and self.use_mean:
-            self.reference_ = X.mean(axis=0)
+        if self.reference is not None:
+            self.reference_ = np.array(self.reference)
+            self.A_ = self._calculate_A(self.reference_)
+            self.weights_ = np.array(self.weights)
             return self
 
-        if self.reference is None and self.use_median:
+        if self.use_median:
             self.reference_ = np.median(X, axis=0)
+            self.A_ = self._calculate_A(self.reference_)
+            self.weights_ = np.array(self.weights)
             return self
 
-        if self.reference is not None:
-            self.reference_ = self.reference.copy()
+        if self.use_mean:
+            self.reference_ = X.mean(axis=0)
+            self.A_ = self._calculate_A(self.reference_)
+            self.weights_ = np.array(self.weights)
             return self
 
         raise ValueError("No reference was provided")
@@ -111,13 +135,33 @@ class MultiplicativeScatterCorrection(OneToOneFeatureMixin, BaseEstimator, Trans
         X = check_input(X)
         X_ = X.copy()
 
+        # Check that the number of features is the same as the fitted data
+        if X_.shape[1] != self.n_features_in_:
+            raise ValueError(
+                f"Expected {self.n_features_in_} features but got {X_.shape[1]}"
+            )
+
         # Calculate the multiplicative signal correction
-        ones = np.ones(X.shape[1])
-        for i, x in enumerate(X_):
-            X_[i] = self._calculate_multiplicative_correction(x, ones)
-        return X_.reshape(-1, 1) if X_.ndim == 1 else X_
-
-    def _calculate_multiplicative_correction(self, x, ones) -> np.ndarray:
-        A = np.vstack([self.reference_, ones]).T
-        m, c = np.linalg.lstsq(A, x, rcond=None)[0]
+        if self.weights is None:
+            for i, x in enumerate(X_):
+                X_[i] = self._calculate_multiplicative_correction(x)
+            return X_.reshape(-1, 1) if X_.ndim == 1 else X_
+
+        if self.weights is not None:
+            for i, x in enumerate(X_):
+                X_[i] = self._calculate_weighted_multiplicative_correction(x)
+            return X_.reshape(-1, 1) if X_.ndim == 1 else X_
+
+    def _calculate_weighted_multiplicative_correction(self, x) -> np.ndarray:
+        m, c = np.linalg.lstsq(
+            np.diag(self.weights_) @ self.A_, x * self.weights_, rcond=None
+        )[0]
         return (x - c) / m
+
+    def _calculate_multiplicative_correction(self, x) -> np.ndarray:
+        m, c = np.linalg.lstsq(self.A_, x, rcond=None)[0]
+        return (x - c) / m
+
+    def _calculate_A(self, reference):
+        ones = np.ones(reference.shape[0])
+        return np.vstack([reference, ones]).T

chemotools/scatter/robust_normal_variate.py

@@ -0,0 +1,109 @@
+import numpy as np
+from sklearn.base import BaseEstimator, TransformerMixin, OneToOneFeatureMixin
+from sklearn.utils.validation import check_is_fitted
+
+from chemotools.utils.check_inputs import check_input
+
+
+class RobustNormalVariate(OneToOneFeatureMixin, BaseEstimator, TransformerMixin):
+    """
+    A transformer that calculates the robust normal variate of the input data.
+
+    Parameters
+    ----------
+    percentile : float, optional
+        The percentile to use for the robust normal variate. The value should be
+        between 0 and 100. The default is 25.
+
+    Attributes
+    ----------
+    n_features_in_ : int
+        The number of features in the input data.
+
+    _is_fitted : bool
+        Whether the transformer has been fitted to data.
+
+    Methods
+    -------
+    fit(X, y=None)
+        Fit the transformer to the input data.
+
+    transform(X, y=0, copy=True)
+        Transform the input data by calculating the standard normal variate.
+
+    References
+    ----------
+    Q. Guo, W. Wu, D.L. Massart. The robust normal variate transform for pattern
+    recognition with near-infrared data. doi:10.1016/S0003-2670(98)00737-5
+    """
+
+    def __init__(self, percentile: float = 25):
+        self.percentile = percentile
+
+    def fit(self, X: np.ndarray, y=None) -> "RobustNormalVariate":
+        """
+        Fit the transformer to the input data.
+
+        Parameters
+        ----------
+        X : np.ndarray of shape (n_samples, n_features)
+            The input data to fit the transformer to.
+
+        y : None
+            Ignored.
+
+        Returns
+        -------
+        self : RobustNormalVariate
+            The fitted transformer.
+        """
+        # Check that X is a 2D array and has only finite values
+        X = check_input(X)
+
+        # Set the number of features
+        self.n_features_in_ = X.shape[1]
+
+        # Set the fitted attribute to True
+        self._is_fitted = True
+
+        return self
+
+    def transform(self, X: np.ndarray, y=None) -> np.ndarray:
+        """
+        Transform the input data by calculating the standard normal variate.
+
+        Parameters
+        ----------
+        X : np.ndarray of shape (n_samples, n_features)
+            The input data to transform.
+
+        y : None
+            Ignored.
+
+        Returns
+        -------
+        X_ : np.ndarray of shape (n_samples, n_features)
+            The transformed data.
+        """
+        # Check that the estimator is fitted
+        check_is_fitted(self, "_is_fitted")
+
+        # Check that X is a 2D array and has only finite values
+        X = check_input(X)
+        X_ = X.copy()
+
+        # Check that the number of features is the same as the fitted data
+        if X_.shape[1] != self.n_features_in_:
+            raise ValueError(
+                f"Expected {self.n_features_in_} features but got {X_.shape[1]}"
+            )
+
+        # Calculate the standard normal variate
+        for i, x in enumerate(X_):
+            X_[i] = self._calculate_robust_normal_variate(x)
+
+        return X_.reshape(-1, 1) if X_.ndim == 1 else X_
+
+    def _calculate_robust_normal_variate(self, x) -> np.ndarray:
+        percentile = np.percentile(x, self.percentile)
+        return (x - percentile) / np.std(x[x <= percentile])

{chemotools-0.0.22.dist-info → chemotools-0.0.24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: chemotools
-Version: 0.0.22
+Version: 0.0.24
 Summary: Package to integrate chemometrics in scikit-learn pipelines
 Home-page: https://github.com/paucablop/chemotools
 Author: Pau Cabaneros Lopez
@@ -13,6 +13,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy
+Requires-Dist: pandas
 Requires-Dist: scipy
 Requires-Dist: scikit-learn
 

{chemotools-0.0.22.dist-info → chemotools-0.0.24.dist-info}/RECORD

@@ -8,16 +8,24 @@ chemotools/baseline/linear_correction.py,sha256=6Sw2n4QTvIDKWRdJpFD48hMvOEwqbctU
 chemotools/baseline/non_negative.py,sha256=17_82l95U9kgoQ3Pdz3-jGv8B51JzqPdHODt6PegWRw,2864
 chemotools/baseline/polynomial_correction.py,sha256=caP866fwZb7PASyz6oezgg8hdZtFMT0EimK89TGSTSc,4059
 chemotools/baseline/subtract_reference.py,sha256=Pht87XadXK0URq2fun66OHaUk_cx56AkF84ta3VJy_8,3441
+chemotools/datasets/__init__.py,sha256=yarhf-7bKB-mbStdWfi9LA_apOusoxY5A9bcwyzj10M,85
+chemotools/datasets/_base.py,sha256=ArZrVRW5m5yO13iK_EycvV8gheiWKR9hoSZCD_OfS1g,2249
+chemotools/datasets/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+chemotools/datasets/data/fermentation_hplc.csv,sha256=o80j8kQGBgmzmwrqDP9bmMxQoeEgtIu6q3Yng7VUKyk,2090
+chemotools/datasets/data/fermentation_spectra.csv,sha256=MaaNMQP0lygJgFbEoUX0OUqdA-id8mF5Llvf_vj9tJk,15237508
+chemotools/datasets/data/train_hplc.csv,sha256=DjtmqiePOWB-F6TsOGFngE1pKyXkb7Xmsi-1CLxsTnE,249
+chemotools/datasets/data/train_spectra.csv,sha256=iVF19W52NHlbqq8BbLomn8n47kSPT0QxJv7wtQX4yjQ,203244
 chemotools/derivative/__init__.py,sha256=x2F0IJ-uCbEYFoXFbZl_RTPCbSq82vqGOwlM9R_2Klo,84
-chemotools/derivative/norris_william.py,sha256=M7Uax3Aq44CbqmRO0GvP34NVxfRr9NYh5QRwhIAjpNU,5015
+chemotools/derivative/norris_william.py,sha256=JaJ7zlSiC_0tiITu7VWXtgKrmkQP7gLvuFb0_n1j9Dw,5081
 chemotools/derivative/savitzky_golay.py,sha256=fFzQRVGVXQIUkHp1x9dqfLVPlyStubIhSj9aGfZKuXY,3745
 chemotools/scale/__init__.py,sha256=qRDhHXhkwXrr0a9ctKVpjp8X8H8Wcu2pelavehv-8ik,115
 chemotools/scale/index_scaler.py,sha256=GsSVEfhVud-ZSVF7YwJBbix976W4a-1SXtbjUtQdqZ4,2661
 chemotools/scale/min_max_scaler.py,sha256=zjhPhP5PcLh796VhNxo73INutGkUThe08B6IxMVD3X8,2850
 chemotools/scale/norm_scaler.py,sha256=qNs-npf5Jqcp8RYqt88_5-zwd-yIo-J1jItgUTFeozs,2699
-chemotools/scatter/__init__.py,sha256=n47sqTpKK4N8bt6howklP8Z2ceZrkASTqfztwfGlEfc,137
-chemotools/scatter/extended_multiplicative_scatter_correction.py,sha256=Xe2tEF06zXe4Kuox9NyR6e2jkCBx4zkZaJc9zgwxngE,1073
-chemotools/scatter/multiplicative_scatter_correction.py,sha256=Qme8lSQdyaUmQ2vbtVUTpOkm5tZvDHBCOuPlQ3dGMIU,3955
+chemotools/scatter/__init__.py,sha256=M0_B4hXVoDc2Qx00QreUfhFqPUTs6LbU4CWaFU17hg4,288
+chemotools/scatter/extended_multiplicative_scatter_correction.py,sha256=J65hyEFBzKNo_35Ta9MKWO35CjTw-8hDbSr8xd8RIfc,6912
+chemotools/scatter/multiplicative_scatter_correction.py,sha256=MFemiwS-KWFOtlcXVhLnY4mn6QQ8pttuj6UP0rodXEM,5689
+chemotools/scatter/robust_normal_variate.py,sha256=joIL-nGUja0nG8YcCuT32ehxmy2xOy3OD0t0yP5vWfM,3233
 chemotools/scatter/standard_normal_variate.py,sha256=wmK_8ea2CvoLaGebBFKr8zAU7QjGbaKAg04y6iZ4sDc,2681
 chemotools/smooth/__init__.py,sha256=Kwg3jVnl-W-efTHMR6-6hQsTp-An1lYQ1lZFj6sNMtg,176
 chemotools/smooth/mean_filter.py,sha256=fcC4EjO57Br3I9SJqWDJRxPxAv2WjjmXTECdBmBYXLI,2953
@@ -30,10 +38,11 @@ chemotools/variable_selection/__init__.py,sha256=E5WmqGRkM6XgzmhTolP3Tu9KyCtEDk_
 chemotools/variable_selection/range_cut.py,sha256=Gh6flGp616k8gFBNBzxjfz49lncrFulMWukTOPExDTg,3709
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/fixtures.py,sha256=Xa-Vd62Kd1fyWg3PLUSP6iIkOK8etrbyOkMJTn3dvX8,1933
-tests/test_functionality.py,sha256=LsoRlbcWYeK2DJD3Z6S6mo6omFHKCw7dFhzO_jBuP0I,10557
-tests/test_sklearn_compliance.py,sha256=IBsAQ-U5vo11ezfiutXY5HU9DoorxIkTF_6EXa2Fwbc,4100
-chemotools-0.0.22.dist-info/LICENSE,sha256=qtyOy2wDQVX9hxp58h3T-6Lmfv-mSCHoSRkcLUdM9bg,1070
-chemotools-0.0.22.dist-info/METADATA,sha256=W3QxPFeaCLaiG9-YOXKwnAJs40DoPRA7sLGURnSdOxQ,4993
-chemotools-0.0.22.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-chemotools-0.0.22.dist-info/top_level.txt,sha256=eNcNcKSdo-1H_2gwSDrS__dr7BM3R73Cnn-pBiW5FEw,17
-chemotools-0.0.22.dist-info/RECORD,,
+tests/test_datasets.py,sha256=QwqZPLTcXG8f5ZeUJs5bq39v3kVnwSVxPRZ28spobUI,736
+tests/test_functionality.py,sha256=tlvzKSh1OVbWAmSE4L1bPOVN3I205-D3x7a_x2jxpPg,15462
+tests/test_sklearn_compliance.py,sha256=93RMkmqk4uhuz_wXIntPKCerxOxkQaAzJQwpDL57EaA,4593
+chemotools-0.0.24.dist-info/LICENSE,sha256=qtyOy2wDQVX9hxp58h3T-6Lmfv-mSCHoSRkcLUdM9bg,1070
+chemotools-0.0.24.dist-info/METADATA,sha256=gd8kokfQExRBNZgurl6vCI6O-j3QVviwYBfvisuKRd4,5015
+chemotools-0.0.24.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+chemotools-0.0.24.dist-info/top_level.txt,sha256=eNcNcKSdo-1H_2gwSDrS__dr7BM3R73Cnn-pBiW5FEw,17
+chemotools-0.0.24.dist-info/RECORD,,

{chemotools-0.0.22.dist-info → chemotools-0.0.24.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.40.0)
+Generator: bdist_wheel (0.41.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

tests/test_datasets.py

@@ -0,0 +1,30 @@
+import pandas as pd
+
+from chemotools.datasets import load_fermentation_test, load_fermentation_train
+
+
+def test_load_fermentation_test():
+    # Arrange
+
+    # Act
+    test_spectra, test_hplc = load_fermentation_test()
+
+    # Assert
+    assert test_spectra.shape == (1629, 1047)
+    assert test_hplc.shape == (34, 6)
+    assert isinstance(test_spectra, pd.DataFrame)
+    assert isinstance(test_hplc, pd.DataFrame)
+
+def test_load_fermentation_train():
+    # Arrange
+
+    # Act
+    train_spectra, train_hplc = load_fermentation_train()
+
+    # Assert
+    assert train_spectra.shape == (21, 1047)
+    assert train_hplc.shape == (21, 1)
+    assert isinstance(train_spectra, pd.DataFrame)
+    assert isinstance(train_hplc, pd.DataFrame)
+
+