mediml 0.9.9__py3-none-any.whl

MEDiml/filters/utils.py

@@ -0,0 +1,107 @@
+from typing import List
+
+import numpy as np
+from scipy.signal import fftconvolve
+
+
+def pad_imgs(
+            images: np.ndarray,
+            padding_length: List,
+            axis: List,
+            mode: str
+            )-> np.ndarray:
+    """Apply padding on a 3d images using a 2D padding pattern.
+
+    Args:
+        images (ndarray): a numpy array that represent the image.
+        padding_length (List): The padding length that will apply on each side of each axe.
+        axis (List): A list of axes on which the padding will be done.
+        mode (str): The padding mode. Check options here: `numpy.pad 
+            <https://numpy.org/doc/stable/reference/generated/numpy.pad.html>`__.
+
+    Returns:
+        ndarray: A numpy array that represent the padded image.
+    """
+    pad_tuple = ()
+    j = 1
+
+    for i in range(np.ndim(images)):
+        if i in axis:
+            pad_tuple += ((padding_length[-j], padding_length[-j]),)
+            j += 1
+        else:
+            pad_tuple += ((0, 0),)
+
+    return np.pad(images, pad_tuple, mode=mode)
+
+def convolve(
+        dim: int,
+        kernel: np.ndarray,
+        images: np.ndarray,
+        orthogonal_rot: bool=False,
+        mode: str = "symmetric"
+    ) -> np.ndarray:
+    """Convolve a given n-dimensional array with the kernel to generate a filtered image.
+
+    Args:
+        dim (int): The dimension of the images.
+        kernel (ndarray): The kernel to use for the convolution.
+        images (ndarray): A n-dimensional numpy array that represent a batch of images to filter.
+        orthogonal_rot (bool, optional): If true, the 3D images will be rotated over coronal, axial and sagittal axis.
+        mode (str, optional): The padding mode. Check options here: `numpy.pad 
+            <https://numpy.org/doc/stable/reference/generated/numpy.pad.html>`__.
+
+    Returns:
+        ndarray: The filtered image.
+    """
+
+    in_size = np.shape(images)
+
+    # We only handle 2D or 3D images.
+    assert len(in_size) == 3 or len(in_size) == 4, \
+        "The tensor should have the followed shape (B, H, W) or (B, D, H, W)"
+
+    if not orthogonal_rot:
+        # If we have a 2D kernel but a 3D images, we squeeze the tensor
+        if dim < len(in_size) - 1:
+            images = images.reshape((in_size[0] * in_size[1], in_size[2], in_size[3]))
+
+        # We compute the padding size along each dimension
+        padding = [int((kernel.shape[-1] - 1) / 2) for _ in range(dim)]
+        pad_axis_list = [i for i in range(1, dim+1)]
+
+        # We pad the images and we add the channel axis.
+        padded_imgs = pad_imgs(images, padding, pad_axis_list, mode)
+        new_imgs = np.expand_dims(padded_imgs, axis=1)
+
+        # Operate the convolution
+        if dim < len(in_size) - 1:
+            # If we have a 2D kernel but a 3D images, we convolve slice by slice
+            result_list = [fftconvolve(np.expand_dims(new_imgs[i], axis=0), kernel, mode='valid') for i in range(len(images))]
+            result = np.squeeze(np.stack(result_list), axis=2)
+
+        else :
+            result = fftconvolve(new_imgs, kernel, mode='valid')
+
+        # Reshape the data to retrieve the following format: (B, C, D, H, W)
+        if dim < len(in_size) - 1:
+            result = result.reshape((
+                in_size[0], in_size[1], result.shape[1], in_size[2], in_size[3])
+            ).transpose(0, 2, 1, 3, 4)
+
+    # If we want orthogonal rotation
+    else:
+        coronal_imgs = images
+        axial_imgs, sagittal_imgs = np.rot90(images, 1, (1, 2)), np.rot90(images, 1, (1, 3))
+        
+        result_coronal = convolve(dim, kernel, coronal_imgs, False, mode)
+        result_axial = convolve(dim, kernel, axial_imgs, False, mode)
+        result_sagittal = convolve(dim, kernel, sagittal_imgs, False, mode)
+
+        # split and unflip and stack the result on a new axis
+        result_axial = np.rot90(result_axial, 1, (3, 2))
+        result_sagittal = np.rot90(result_sagittal, 1, (4, 2))
+
+        result = np.stack([result_coronal, result_axial, result_sagittal])
+
+    return result

MEDiml/filters/wavelet.py

@@ -0,0 +1,237 @@
+import math
+from itertools import combinations, permutations
+from typing import List, Union
+
+import numpy as np
+import pywt
+
+from ..MEDscan import MEDscan
+from ..utils.image_volume_obj import image_volume_obj
+
+
+class Wavelet():
+    """
+    The wavelet filter class.
+    """
+
+    def __init__(
+            self,
+            ndims: int,
+            wavelet_name="haar",
+            padding="symmetric",
+            rot_invariance=False):
+        """The constructor of the wavelet filter
+
+        Args:
+            ndims (int): The number of dimension of the images that will be filter as int.
+            wavelet_name (str): The name of the wavelet kernel as string.
+            padding (str): The padding type that will be used to produce the convolution
+            rot_invariance (bool): If true, rotation invariance will be done on the images.
+
+        Returns:
+            None
+        """
+        self.dim = ndims
+        self.padding = padding
+        self.rot = rot_invariance
+        self.wavelet = None
+        self.kernel_length = None
+        self.create_kernel(wavelet_name)
+
+    def create_kernel(self,
+                      wavelet_name: str):
+        """Get the wavelet object and his kernel length.
+
+        Args:
+            wavelet_name (str): A string that represent the wavelet name that will be use to create the kernel
+
+        Returns:
+            None
+        """
+
+        self.wavelet = pywt.Wavelet(wavelet_name)
+        self.kernel_length = max(self.wavelet.rec_len, self.wavelet.dec_len)
+
+    def __unpad(self,
+                images: np.ndarray,
+                padding: List) -> np.ndarray:
+        """Unpad a batch of images
+
+        Args:
+            images: A numpy nd-array or a list that represent the batch of padded images.
+                    The shape should be (B, H, W) or (B, H, W, D)
+            padding: a list of length 2*self.dim that gives the length of padding on each side of each axis.
+
+        Returns: 
+            ndarray: A numpy nd-array or a list that represent the batch of unpadded images
+        """
+
+        if self.dim == 2:
+            return images[:, padding[0]:-padding[1], padding[2]:-padding[3]]
+        elif self.dim == 3:
+            return images[:, padding[0]:-padding[1], padding[2]:-padding[3], padding[4]:-padding[5]]
+        else:
+            raise NotImplementedError
+
+    def __get_pad_length(self,
+                         image_shape: List,
+                         level: int) -> np.ndarray:
+        """Compute the padding length needed to have a padded image where the length
+        along each axis is a multiple 2^level.
+
+        Args:
+            image_shape (List): a list of integer that describe the length of the image along each axis.
+            level (int): The level of the wavelet transform
+
+        Returns: 
+            ndarray: An integer list of length 2*self.dim that gives the length of padding on each side of each axis.
+        """
+        padding = []
+        ker_length = self.kernel_length*level
+        for l in image_shape:
+            padded_length = math.ceil((l + 2*(ker_length-1)) / 2**level) * 2**level - l
+            padding.extend([math.floor(padded_length/2), math.ceil(padded_length/2)])
+
+        return padding
+
+    def _pad_imgs(self,
+                  images: np.ndarray,
+                  padding,
+                  axis: List):
+        """Apply padding on a 3d images using a 2D padding pattern (special for wavelet).
+
+        Args:
+            images: a numpy array that represent the image.
+            padding: The padding length that will apply on each side of each axe.
+            axis: A list of axes on which the padding will be done.
+
+        Returns: 
+            ndarray: A numpy array that represent the padded image.
+        """
+        pad_tuple = ()
+        j = 0
+
+        for i in range(np.ndim(images)):
+            if i in axis:
+                pad_tuple += ((padding[j], padding[j+1]),)
+                j += 2
+            else:
+                pad_tuple += ((0, 0),)
+
+        return np.pad(images, pad_tuple, mode=self.padding)
+        
+
+    def convolve(self,
+                 images: np.ndarray,
+                 _filter="LHL",
+                 level=1)-> np.ndarray:
+        """Filter a given batch of images using pywavelet.
+
+        Args:
+            images (ndarray): A n-dimensional numpy array that represent the images to filter
+            _filter (str): The filter to uses.
+            level (int): The number of decomposition steps to perform.
+
+        Returns:
+            ndarray: The filtered image as numpy nd-array
+        """
+
+        # We pad the images
+        padding = self.__get_pad_length(np.shape(images[0]), level)
+        axis_list = [i for i in range(0, self.dim)]
+        images = np.expand_dims(self._pad_imgs(images[0], padding, axis_list), axis=0)
+
+        # We generate the to collect the result from pywavelet dictionary
+        _index = str().join(['a' if _filter[i] == 'L' else 'd' for i in range(len(_filter))])
+
+        if self.rot:
+            result = []
+            _index_list = np.unique([str().join(perm) for perm in permutations(_index, self.dim)])
+
+            # For each images, we flip each axis.
+            for image in images:
+                axis_rot = [comb for j in range(self.dim+1) for comb in combinations(np.arange(self.dim), j)]
+                images_rot = [np.flip(image, axis) for axis in axis_rot]
+
+                res_rot = []
+                for i in range(len(images_rot)):
+                    filtered_image = pywt.swtn(images_rot[i], self.wavelet, level=level)[0]
+                    res_rot.extend([np.flip(filtered_image[j], axis=axis_rot[i]) for j in _index_list])
+
+                result.extend([np.mean(res_rot, axis=0)])
+        else:
+            result = []
+            for i in range(len(images)):
+                result.extend([pywt.swtn(images[i], self.wavelet, level=level)[level-1][_index]])
+
+        return self.__unpad(np.array(result), padding)
+
+def apply_wavelet(
+        input_images: Union[np.ndarray, image_volume_obj],
+        medscan: MEDscan = None,
+        ndims: int = 3,
+        wavelet_name: str = "haar",
+        subband: str = "LHL",
+        level: int = 1,
+        padding: str = "symmetric",
+        rot_invariance: bool = False
+    ) -> np.ndarray:
+    """Apply the mean filter to the input image
+    
+    Args:
+        input_images (ndarray): The image to filter.
+        medscan (MEDscan, optional): The MEDscan object that will provide the filter parameters.
+        ndims (int, optional): The number of dimensions of the input image.
+        wavelet_name (str): The name of the wavelet kernel as string.
+        level (List[str], optional): The number of decompositions steps to perform.
+        subband (str, optional): String of the 1D wavelet kernels ("H" for high-pass 
+            filter or "L" for low-pass filter). Must have a size of ``ndims``.
+        padding (str, optional): The padding type that will be used to produce the convolution. Check options 
+            here: `numpy.pad <https://numpy.org/doc/stable/reference/generated/numpy.pad.html>`__.
+        rot_invariance (bool, optional): If true, rotation invariance will be done on the kernel.
+
+    Returns:
+        ndarray: The filtered image.
+    """
+    # Check if the input is a numpy array or a Image volume object
+    spatial_ref = None
+    if type(input_images) == image_volume_obj:
+        spatial_ref = input_images.spatialRef
+        input_images = input_images.data
+    
+    # Convert to shape : (B, W, H, D)
+    input_images = np.expand_dims(input_images.astype(np.float64), axis=0) 
+
+    if medscan:
+        # Initialize filter class instance
+        _filter = Wavelet(
+                        ndims=medscan.params.filter.wavelet.ndims, 
+                        wavelet_name=medscan.params.filter.wavelet.basis_function,
+                        rot_invariance=medscan.params.filter.wavelet.rot_invariance,
+                        padding=medscan.params.filter.wavelet.padding
+                    )
+        # Run convolution
+        result = _filter.convolve(
+                        input_images, 
+                        _filter=medscan.params.filter.wavelet.subband, 
+                        level=medscan.params.filter.wavelet.level
+                    )
+    else:
+        # Initialize filter class instance
+        _filter = Wavelet(
+                        ndims=ndims, 
+                        wavelet_name=wavelet_name,
+                        rot_invariance=rot_invariance,
+                        padding=padding
+                    )
+        # Run convolution
+        result = _filter.convolve(
+                        input_images, 
+                        _filter=subband, 
+                        level=level
+                    )
+    
+    if spatial_ref:
+        return image_volume_obj(np.squeeze(result), spatial_ref)
+    else:
+        return np.squeeze(result)

MEDiml/learning/DataCleaner.py

@@ -0,0 +1,198 @@
+import random
+from typing import Dict, List
+
+import numpy as np
+import pandas as pd
+
+
+class DataCleaner:
+    """
+    Class that will clean features of the csv by removing features with too many missing values,
+    too little variation, too many missing values per sample, too little variation per sample,
+    and imputing missing values.
+    """
+    def __init__(self, df_features: pd.DataFrame, type: str = "continuous"):
+        """
+        Constructor of the class DataCleaner
+
+        Args:
+            df_features (pd.DataFrame): Table of features.
+            type (str): Type of variable: "continuous", "hcategorical" or "icategorical". Defaults to "continuous".
+        """
+        self.df_features = df_features
+        self.type = type
+    
+    def __update_df_features(self, var_of_type: List[str], flag_var_out: List[bool]) -> List[str]:
+        """
+        Updates the variable table by deleting the features that are not in the variable of type
+
+        Args:
+            var_of_type (List[str]): List of variable names.
+            flag_var_out (List[bool]): List of variables to flag out.
+        
+        Returns:
+            List[str]: List of variable names that were not flagged out.
+        """
+        var_to_delete = np.delete(var_of_type, [i for i, v in enumerate(flag_var_out) if not v])
+        var_of_type = np.delete(var_of_type, [i for i, v in enumerate(flag_var_out) if v])
+        self.df_features = self.df_features.drop(var_to_delete, axis=1)
+        return var_of_type
+
+    def cut_off_missing_per_sample(self, var_of_type: List[str], missing_cutoff : float = 0.25) -> None:
+        """
+        Removes observations/samples with more than ``missing_cutoff`` missing features.
+
+        Args:
+            var_of_type (List[str]): List of variable names.
+            missing_cutoff (float): Maximum percentage cut-offs of missing features per sample. Defaults to 25%.
+        
+        Returns:
+            None.
+        """
+        # Initialization
+        n_observation, n_features = self.df_features.shape
+        empty_vec = np.zeros(n_observation, dtype=int)
+        data = self.df_features[var_of_type]
+        empty_vec += data.isna().sum(axis=1).values
+        
+        # Gathering results
+        ind_obs_out = np.where(((empty_vec/n_features) > missing_cutoff) == True)
+        self.df_features = self.df_features.drop(self.df_features.index[ind_obs_out])
+    
+    def cut_off_missing_per_feature(self, var_of_type: List[str], missing_cutoff : float = 0.1) -> List[str]:
+        """
+        Removes features with more than ``missing_cutoff`` missing patients.
+
+        Args:
+            var_of_type (list): List of variable names.
+            missing_cutoff (float): maximal percentage cut-offs of missing patient samples per variable.
+        
+        Returns:
+            List[str]: List of variable names that were not flagged out.
+        """
+        flag_var_out = (((self.df_features[var_of_type].isna().sum()) / self.df_features.shape[0]) > missing_cutoff)
+        return self.__update_df_features(var_of_type, flag_var_out)
+
+    def cut_off_variation(self, var_of_type: List[str], cov_cutoff : float = 0.1) -> List[str]:
+        """
+        Removes features with a coefficient of variation (cov) less than ``cov_cutoff``.
+
+        Args:
+            var_of_type (list): List of variable names.
+            cov_cutoff (float): minimal coefficient of variation cut-offs over samples per variable. Defaults to 10%.
+        
+        Returns:
+            List[str]: List of variable names that were not flagged out.
+        """
+        eps = np.finfo(np.float32).eps
+        cov_df_features = (self.df_features[var_of_type].std(skipna=True) / self.df_features[var_of_type].mean(skipna=True))
+        flag_var_out = cov_df_features.abs().add(eps) < cov_cutoff
+        return self.__update_df_features(var_of_type, flag_var_out)
+    
+    def impute_missing(self, var_of_type: List[str], imputation_method : str = "mean") -> None:
+        """
+        Imputes missing values of the features of type.
+
+        Args:
+            var_of_type (list): List of variable names.
+            imputation_method (str): Method of imputation. Can be "mean", "median", "mode" or "random".
+                For "random" imputation, a seed can be provided by adding the seed value after the method 
+                name, for example "random42".
+        
+        Returns:
+            None.
+        """
+        if self.type in ['continuous', 'hcategorical']:
+            # random imputation
+            if 'random' in imputation_method:
+                if len(imputation_method) > 6:
+                    try:
+                        seed = int(imputation_method[7:])
+                        random.seed(seed)
+                    except Exception as e:
+                        print(f"Warning: Seed must be an integer. Random seed will be set to None. str({e})")
+                        random.seed(a=None)
+                else:
+                    random.seed(a=None)
+                self.df_features[var_of_type] = self.df_features[var_of_type].apply(lambda x: x.fillna(random.choice(list(x.dropna(axis=0)))))
+            
+            # Imputation with median
+            elif 'median' in imputation_method:
+                self.df_features[var_of_type] = self.df_features[var_of_type].fillna(self.df_features[var_of_type].median())
+            
+            # Imputation with mean
+            elif 'mean' in imputation_method:
+                self.df_features[var_of_type] = self.df_features[var_of_type].fillna(self.df_features[var_of_type].mean())
+            
+            else:
+                raise ValueError("Imputation method for continuous and hcategorical features must be 'random', 'median' or 'mean'.")
+        
+        elif self.type in ['icategorical']:
+            if 'random' in imputation_method:
+                if len(imputation_method) > 6:
+                    seed = int(imputation_method[7:])
+                    random.seed(seed)
+                else:
+                    random.seed(a=None)
+
+                self.df_features[var_of_type] = self.df_features[var_of_type].apply(lambda x: x.fillna(random.choice(list(x.dropna(axis=0)))))
+
+            if 'mode' in imputation_method:
+                self.df_features[var_of_type] = self.df_features[var_of_type].fillna(self.df_features[var_of_type].mode().max())
+        else:
+            raise ValueError("Variable type must be 'continuous', 'hcategorical' or 'icategorical'.")
+        
+    def __call__(self, cleaning_dict: Dict, imputation_method: str = "mean", 
+                missing_cutoff_ps: float = 0.25, missing_cutoff_pf: float = 0.1, 
+                cov_cutoff:float = 0.1) -> pd.DataFrame:
+        """
+        Applies data cleaning to the features of type.
+
+        Args:
+            cleaning_dict (dict): Dictionary of cleaning parameters (missing cutoffs and coefficient of variation cutoffs etc.).
+            var_of_type (list, optional): List of variable names.
+            imputation_method (str): Method of imputation. Can be "mean", "median", "mode" or "random".
+                For "random" imputation, a seed can be provided by adding the seed value after the method 
+                name, for example "random42".
+            missing_cutoff_ps (float, optional): maximal percentage cut-offs of missing features per sample.
+            missing_cutoff_pf (float, optional): maximal percentage cut-offs of missing samples per variable.
+            cov_cutoff (float, optional): minimal coefficient of variation cut-offs over samples per variable.
+        
+        Returns:
+            pd.DataFrame: Cleaned table of features.
+        """
+
+        # Initialization
+        var_of_type = self.df_features.Properties['userData']['variables']['continuous']
+
+        # Retrieve thresholds from cleaning_dict if not None
+        if cleaning_dict is not None:
+            missing_cutoff_pf = cleaning_dict['missingCutoffpf']
+            missing_cutoff_ps = cleaning_dict['missingCutoffps']
+            cov_cutoff = cleaning_dict['covCutoff']
+            imputation_method = cleaning_dict['imputation']
+        
+        # Replace infinite values with NaNs
+        self.df_features = self.df_features.replace([np.inf, -np.inf], np.nan)
+
+        # Remove features with more than missing_cutoff_pf missing samples (NaNs)
+        var_of_type = self.cut_off_missing_per_feature(var_of_type, missing_cutoff_pf)
+
+        # Check
+        if len(var_of_type) == 0:
+            return None
+
+        # Remove features with a coefficient of variation less than cov_cutoff
+        var_of_type = self.cut_off_variation(var_of_type, cov_cutoff)
+
+        # Check
+        if len(var_of_type) == 0:
+            return None
+
+        # Remove scans with more than missing_cutoff_ps missing features
+        self.cut_off_missing_per_sample(var_of_type, missing_cutoff_ps)
+
+        # Impute missing values
+        self.impute_missing(var_of_type, imputation_method)
+
+        return self.df_features