pico-ml 2.0.0__py3-none-any.whl

pico/domain/SplitGroup.py

@@ -0,0 +1,202 @@
+from typing import List, Tuple, Union
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+
+from . import MetaData
+from ..service import Utils, init_logger
+import numpy as np
+
+
+class SplitGroup:
+    def __init__(self, metadata: MetaData, selected_targets: List[str], train_test_proportion: float,
+                 number_of_splits: int, classes_design: dict, pairing_column: str, 
+                 uniq_sample_id: List[str], balance_correction: int = 0,
+                 classes_repartition: Union[dict, None] = None,
+                 test_split_seed: Union[int,None] = None):
+        self._logger = init_logger()
+        self._metadata = metadata
+        self._number_of_split = number_of_splits
+        self._classes_design = classes_design
+        self._splits = []
+        self._compute_splits(train_test_proportion, number_of_splits, pairing_column, selected_targets,
+                             uniq_sample_id, balance_correction, classes_repartition, test_split_seed)
+
+    def _compute_splits(self, train_test_proportion: float, number_of_splits: int, pairing_column: str,
+                        selected_targets: List[str],  uniq_sample_id: List[str], 
+                        balance_correction: int = 0,
+                        classes_repartition: Union[dict, None] = None,
+                        test_split_seed: Union[int,None] = None) -> None:
+        """
+        Create the desired number of split for the experiment. It includes/hadles the train-test repartition, the class
+        balancing, the pairing of samples, the classes design, etc.
+
+        Args:
+            train_test_proportion (float): Proportion of the dataset to include in the test split.
+            number_of_splits (int): total number of splits
+            pairing_column (str): Column to user of the pairing. When empty ("") not pairing is done.
+            selected_targets (List[str]): The selection of classes done with the interface or the automate.py (the names of the
+                selected classes/targets)
+                We consider selected_targets that has targets coming from multiple columns, that they are separated
+                by "__" i.e. "ali__A", "med__B", etc.
+            balance_correction (int, optional): balance correction to adjust proportion between classes.
+                Defaults to 0. (for no balancing)
+            classes_repartition (Union[dict, None], optional): . Defaults to None.
+            test_split_seed (int | None, optional): Split seed number. For test purpose only,
+                to be used from automate.py to test one specific split. Defaults to None.
+        """
+        self._logger.info("_compute_split function beginning")
+
+        # 1 - filter out the samples having a target not included in the classification design
+        # retrieve metadata dataframe
+        df_filter = self._metadata.get_metadata()
+        # keep only the lines for which the value in the final_targets column is in selected_targets
+        df_filter = df_filter[df_filter[self._metadata.get_target_column()].isin(selected_targets)]
+        # keep only the lines that correspond to samples in data file
+        # (handles the cases of a metadata file for multiple data files : where samples in 
+        # the metadata having corresponding targets are not in the provided data file)
+        df_filter = df_filter[df_filter[self._metadata.get_id_column()].isin(uniq_sample_id)]
+
+        # 2 - select only one sample per entity
+        if pairing_column != "":
+            # sort the dataframe by the pairing_column values
+            df_entity = df_filter.sort_values(pairing_column)
+            # group samples by the pairing column and keep only the first row of each group (.nth(0) is more stable
+            # than .first())
+            # Carefull : the groupby function change the index of the dataframe to the column it groups by
+            df_entity = df_entity.groupby(pairing_column).nth(0)
+        else:
+            df_entity = df_filter
+
+        # 2.5 - extract ids and targets, transform targets to labels
+        ids = df_entity[self._metadata.get_id_column()]
+        targets = df_entity[self._metadata.get_target_column()]
+        labels = Utils.load_classes_from_targets(self._classes_design, targets)
+
+        # 3- procede with the train-test division on the selected samples        
+        if test_split_seed is not None:
+            self._logger.debug(f"Testing split seed #{test_split_seed}")
+            split_indexes: list[int] = [test_split_seed] # Test only one split seed
+        else:
+            split_indexes = list(range(number_of_splits)) # All splits indexes
+            
+        for split_index in split_indexes:
+            if pairing_column == "":
+                X_train, X_test, y_train, y_test = train_test_split(ids, labels, test_size=train_test_proportion,
+                                                                    random_state=split_index, stratify=labels)
+
+                # 4- retrieve the paired samples corresponding to the one in train or test set
+            else:
+                # random shuffle initialisation for second shuffle of samples
+                rng = np.random.default_rng(seed=split_index)
+                # define the ids column as the index of the dataframe, so it can be extracted with groupby().groups
+                df = df_filter.set_index(self._metadata.get_id_column())
+                # groups is a dictionary with 'keys' as the pairing value and 'values' as the index of the lines corresponding to the pairing
+                groups = df.groupby(pairing_column).groups
+                # apply the train-test division on the pairing values / the entity
+                # TODO : careful check if labels is in the right order with the data
+                X_train_temp, X_test_temp, y_train_temp, y_test_temp = train_test_split(df_entity.index, labels,
+                                                                                        test_size=train_test_proportion,
+                                                                                        random_state=split_index,
+                                                                                        stratify=labels)
+                # retrieve the ids corresponding the to entities in train
+                X_train = []
+                for representative in X_train_temp:
+                    represented_pairing_value = df_filter.loc[representative][pairing_column]
+                    X_train.extend(groups[represented_pairing_value])
+                # retrieve targets corresponding to ids and then convert to labels
+                X_train = pd.Series(X_train)
+                targets = df.loc[X_train][self._metadata.get_target_column()]
+                y_train = Utils.load_classes_from_targets(self._classes_design, targets)
+
+                training_data = list(zip(X_train, y_train))
+                rng.shuffle(training_data)
+                X_train, y_train = zip(*training_data)
+
+                # retrieve the ids corresponding the to entities in test
+                X_test = []
+                for representative in X_test_temp:
+                    represented_pairing_value = df_filter.loc[representative][pairing_column]
+                    X_test.extend(groups[represented_pairing_value])
+                # retrieve targets corresponding to ids and then convert to labels
+                X_test = pd.Series(X_test)
+                targets = df.loc[X_test][self._metadata.get_target_column()]
+                y_test = Utils.load_classes_from_targets(self._classes_design, targets)
+
+                testing_data = list(zip(X_test, y_test))
+                rng.shuffle(testing_data)
+                X_test, y_test = zip(*testing_data)
+
+            if balance_correction > 0:
+                X_train, y_train = Utils.remove_random_samples_from_class(X_train,
+                                                                          y_train,
+                                                                          balance_correction,
+                                                                          classes_repartition)
+            X_train = list(X_train)
+            y_train = list(y_train)
+            X_test = list(X_test)
+            y_test = list(y_test)
+            
+            if not self._validate_split(y_train, y_test):
+                raise RuntimeError(f"_compute_split step #4 aborted for the invalid split #{split_index}.")
+            
+            self._splits.append([X_train, X_test, y_train, y_test])
+            
+        self._number_of_split = len(self._splits) # Update the number of splits if some have been removed
+        
+        self._logger.info("_compute_split function done")
+
+    def load_split_with_index(self, split_index: int) -> list:
+        return self._splits[split_index]
+
+    def get_number_of_splits(self):
+        """
+        Return the attribute of number of split
+        """
+        return self._number_of_split
+
+    def filter_sample_with_pairing_group(self, pairing_column: str) -> Tuple[List[str], List[str]]:
+        """
+        Function only needs the name of the column used to pair samples together.
+        It retrieves other informations from the attributes (object MetaData).
+        Then it iterates over all the metadata dataframe to store only one sample of each entity (entity stands for a
+        biological source, like an individual). Multiple samples can originate from one entity.
+        """
+        metadata_dataframe = self._metadata.get_metadata()
+        id_column = self._metadata.get_id_column()
+        target_column = self._metadata.get_target_column()
+        filtered_id = []
+        filtered_target = []
+        already_selected_value = set()
+        # TODO : might want to change the process to sorting all lines and then picking the first one
+        for index, row in metadata_dataframe.iterrows():
+            if row[pairing_column] not in already_selected_value:
+                already_selected_value.add(row[pairing_column])
+                filtered_id.append(row[id_column])
+                filtered_target.append(row[target_column])
+        return filtered_id, filtered_target
+
+    def get_selected_targets_and_ids(self, selected_targets: List[str], samples_id: List[str],
+                                     targets: List[str]) -> Tuple[Tuple[str], Tuple[str]]:
+        """
+        Function just filters out the target/id that are not in the selected_targets list
+        """
+        return tuple(zip(*[(target, id) for target, id in zip(targets, samples_id) if target in selected_targets]))
+    
+    def _validate_split(self, y_train: list, y_test: list) -> bool:
+                    # Test and train validation: they must have at least 2 classes.
+        nb_test_classes: int = len(set(y_test))
+        nb_train_classes: int = len(set(y_train))
+        
+        if nb_test_classes < 2 or nb_train_classes < 2:
+            error_msg: str = "At least 2 classes must be present in both train and test splits."
+            if nb_test_classes < 2:
+                error_msg += f" Test set contains only the class '{next(iter(set(y_test)))}'."
+            if nb_train_classes < 2:
+                error_msg += f" Train set contains only the class '{next(iter(set(y_train)))}'."
+            
+            self._logger.error(error_msg)
+            return False
+
+        return True
+

pico/domain/__init__.py

@@ -0,0 +1,9 @@
+from .SplitGroup import SplitGroup
+from .MetaData import MetaData
+from .ClassificationDesign import ClassificationDesign
+from .MetaboExperiment import MetaboExperiment
+from .ModelFactory import ModelFactory
+from .Results import Results
+from .Controller import Controller
+from .DataMatrix import DataMatrix
+from .MetaboModel import MetaboModel

pico/service/DataFormat.py

@@ -0,0 +1,180 @@
+import numpy as np
+import pandas as pd
+import os
+import base64
+import io
+from .Utils import *
+
+
+class DataFormat:
+    """
+    Take data file(s) as input and output a matrix where columns are samples and lines features. With the matrix comes
+    a list of the columns names to retrieve the samples properly.
+    """
+
+    def __init__(self, filename, data=None, use_raw=False, from_base64_str=True):
+        self.use_raw = use_raw
+        self.filename = filename
+        self.base64 = from_base64_str
+        self.data = data
+
+        # TODO : make sure to check if "not progen" matrix are well handled
+        if self.base64:
+            self.in_format = "base64"
+        elif os.path.isfile(filename):
+            self.in_format = "file"
+        elif os.path.isdir(filename):
+            self.in_format = "LDTD"
+        else:
+            raise TypeError(
+                "The given path is not valid, it has to be a file or a directory."
+            )
+
+    def convert(self):
+        """
+        Convert data format for easier use by the PICO
+        """
+        if self.in_format == "base64":
+            data_type, data_string = self.data.split(",")
+            self.data = base64.b64decode(data_string)
+            data = self._convert_from_file()
+        elif self.in_format == "file":
+            data = self._convert_from_file()
+        elif self.in_format == "LDTD":
+            raise ValueError("reading data from LDTD is not supported yet")
+            #data = self._convert_from_LDTD()
+        else:
+            raise ValueError("self.in_format does not correspond to accepted values (base64, file)")
+        return data
+
+    def _convert_from_file(self):
+        """
+        Take a file path or an StringIO object and read it as a pandas Dataframe
+        """
+        file_ext = self.filename.split(".")[-1]
+        # TODO : beware of the sep (, or ;)
+        if "csv" in file_ext:  # Abundance matrices of Progenesis are always in csv format, so its checked first
+            if self.in_format == "base64":  # this condition is to make readable the input data from dcc.Upload
+                self.data = io.StringIO(self.data.decode("utf-8"))
+            else:  # this else is to enable the pd dataframe to be read from full file path
+                self.data = self.filename
+            header = pd.read_csv(self.data, header=None, sep=None, engine="python", nrows=3,
+                                  index_col=0,).fillna("").to_numpy()
+
+            # Needs to reset the pointer to the top of the ioString (to be able to read the string again)
+            if self.in_format == "base64":
+                self.data.seek(0)
+
+            if "Normalised abundance" in header[0] or "Raw abundance" in header[0]:
+                datatable = pd.read_csv(self.data, header=[0, 1, 2], sep=None, engine="python", index_col=0)
+                # Will return : datatable_compoundsInfo, datatable, labels, sample_names
+                return self._read_Progenesis_data_table(datatable, header)
+            else:
+                datatable = pd.read_csv(self.data, sep=None, engine="python", index_col=0)
+                # WARNING : return None, datatable, None, None
+                return self._read_general_data_table(datatable)
+
+        elif ("xls" in file_ext or "od" in file_ext):  # TODO : restrict the "od" condition, might be too large
+            if self.in_format == "base64":  # same as above
+                self.data = io.StringIO(io.BytesIO(self.data))
+            else:
+                self.data = self.filename
+            datatable = pd.read_excel(self.data, index_col=0)
+            # WARNING : return None, datatable, None, None
+            return self._read_general_data_table(datatable)
+
+        else:
+            raise TypeError("The input file is not of the right type, must be excel, odt or csv.")
+
+    def _convert_from_LDTD(self):
+        # TODO :  implement the handling of LDTD data format
+        return ""
+
+    def _read_general_data_table(self, datatable):
+        """
+        for now does nothing, but might be the place to deal with custom format of matrices with extra/unecessary columns
+        or informations
+        ! careful : output only the datable and 3 empty strings because the function that calls it only needs datatable,
+        but that might change
+        """
+        # This return list fits the return when reading a Progenesis file
+        return None, datatable, None, None
+
+    def _read_Progenesis_data_table(self, datatable, header):
+        """
+        Assumes Raw data columns are written after Normalized data columns in the file.
+        :param datatable:
+        :return:
+        """
+        # print(header)
+        if not self.use_raw and "Normalised abundance" in header[0]:  # header.columns.tolist():
+            start_data = list(header[0]).index("Normalised abundance")
+        elif self.use_raw and "Raw abundance" in header[0]:  # header.columns.tolist():
+            start_data = list(header[0]).index("Raw abundance")
+        else:
+            raise KeyError("There is no Raw or Normalized abundance detected in the header.")
+
+        new_header = []
+        for l in header:
+            new_header.append(list_filler(l))
+
+        datatable.columns = new_header
+        datatable_compoundsInfo = datatable.iloc[:, 0:start_data]
+        datatable_compoundsInfo.columns = datatable_compoundsInfo.columns.droplevel([0, 1])
+        datatable_compoundsInfo = datatable_compoundsInfo.T
+
+        if self.use_raw:
+            datatable = datatable["Raw abundance"]
+            labels, sample_names = list(zip(*datatable.columns))
+        else:
+            datatable = datatable["Normalised abundance"]
+            labels, sample_names = list(zip(*datatable.columns))
+
+        datatable.columns = datatable.columns.droplevel(0)
+        datatable = datatable.T
+
+        datatable = datatable.loc[[index for index in datatable.index if "QC" not in index]]
+
+        return datatable_compoundsInfo, datatable, labels, sample_names
+
+        # start_normalized = header.columns.tolist().index("Normalised abundance")
+        # labels_array = np.array(header.iloc[0].tolist())
+
+        # if with_raw:
+        #     start_raw = header.columns.tolist().index("Raw abundance")
+        #     sample_names = datatable.iloc[:, start_normalized:start_raw].columns
+        #     labels = labels_array.tolist()[start_normalized:start_raw]
+        # else:
+        #     sample_names = datatable.iloc[:, start_normalized:].columns
+        #     labels = labels_array.tolist()[start_normalized:]
+        #
+        # current_label = ""
+        # for idx, l in enumerate(labels):
+        #     if l != "nan":
+        #         current_label = l
+        #     else:
+        #         labels[idx] = current_label
+        #
+        # if with_raw:
+        #     datatable_compoundsInfo = datatable.iloc[:, 0:start_normalized]
+        #     datatable_normalized = datatable.iloc[:, start_normalized:start_raw]
+        #     datatable_raw = datatable.iloc[:, start_raw:]
+        #     datatable_raw.columns = [i.rstrip(".1") for i in datatable_raw.columns]  # Fix the columns names
+        #
+        #     datatable_normalized = datatable_normalized.T
+        #     datatable_raw = datatable_raw.T
+        #     datatable_compoundsInfo = datatable_compoundsInfo.T
+        #     datatable_normalized.rename(columns={"Compound": "Sample"})
+        #     datatable_raw.rename(columns={"Compound": "Sample"})
+        #
+        #     if self.use_raw:
+        #         return datatable_compoundsInfo, datatable_raw, labels, sample_names
+        #     else:
+        #         return datatable_compoundsInfo, datatable_normalized, labels, sample_names
+        # else:
+        #     datatable_compoundsInfo = datatable.iloc[:, 0:start_normalized]
+        #     datatable_normalized = datatable.iloc[:, start_normalized:]
+        #     datatable_normalized = datatable_normalized.T
+        #     datatable_compoundsInfo = datatable_compoundsInfo.T
+        #     datatable_normalized.rename(columns={"Compound": "Sample"})
+        #     return datatable_compoundsInfo, datatable_normalized, labels, sample_names

pico/service/ExperimentDesign.py

@@ -0,0 +1,30 @@
+DATA_MATRIX = "Data\\Matrix_normalised_pos.csv"
+
+# Cas, Temoin, TC
+EXPERIMENT_DESIGNS = {
+    "Ctrl_vs_Case": {
+        "classes": {"Controls": ["Temoin"], "Cases": ["Cas"]},
+        "TestSize": 0.2,
+    },
+    # "Control vs TC":{
+    #     "classes": {
+    #         "Controles":["Cas"],
+    #         "TC": ["TC"]
+    #     },
+    #     "TestSize": 0.2,
+    # },
+    # "TC vs Cas":{
+    #     "classes": {
+    #         "TC":["TC"],
+    #         "Cases": ["Cas"]
+    #     },
+    #     "TestSize": 0.2,
+    # },
+    # "Control vs all":{
+    #     "classes": {
+    #         "Control":["Temoin"],
+    #         "All": ["Cas", "TC"]
+    #     },
+    #     "TestSize": 0.2,
+    # },
+}

pico/service/LoggerConfig.py

@@ -0,0 +1,150 @@
+import logging
+import coloredlogs # type: ignore
+from datetime import datetime
+import inspect
+import os
+import threading
+import traceback
+from functools import wraps
+from typing import Callable
+from .Utils import get_pico_subdir
+
+log_filename: str|None = None  # Global variable for log filename
+
+def log_exceptions(logger: logging.Logger) -> Callable:
+    def decorator(func: Callable):
+        @wraps(func)
+        def wrapper(*args, **kwargs) -> Callable:
+            try:
+                return func(*args, **kwargs)
+            except Exception as e:
+                # Log the exception or handle it as needed
+                thread_name = threading.current_thread().name
+                logger.error(f"Error in thread {thread_name}: {e}\n{traceback.format_exc()}")
+                raise # Re-raise the exception to preserve the original
+        return wrapper
+    return decorator
+
+
+def set_log_filename(filename: str="pico.log", add_date: bool=True, level=logging.DEBUG) -> logging.Logger:
+    """Sets the log filename with an optional date suffix.
+    Args:
+        filename (str, optional): The base filename for the log. Defaults to "pico.log".
+        add_date (bool, optional): If True, adds the current date to the filename. Defaults to True.
+    """
+    global log_filename
+
+    if add_date:
+        date_suffix = datetime.now().strftime("%Y-%m-%d")
+        
+        if filename.lower().endswith(".log"):
+            log_filename = filename.replace(".log", f"_{date_suffix}.log")
+        else:
+            log_filename = f"{filename}_{date_suffix}.log"
+    else:
+        log_filename = filename
+
+    # Log file in ~/pico_files/logs directory
+    logs_directory = get_pico_subdir("logs")
+    log_filename = os.path.join(logs_directory, log_filename)      
+
+    # Add "-----------------------" in the log file to start the current session
+    with open(log_filename, 'a') as log_file:
+        if threading.current_thread() is threading.main_thread():
+            log_file.write(f"\n{'----- New start (' + threading.current_thread().name + ') ':-<80}\n")
+        else:
+            log_file.write(f"      New thread ({threading.current_thread().name})\n")
+
+    # Root logger
+    root_logger = logging.getLogger()
+    root_logger.handlers = []  # Clear existing handlers
+    root_logger.setLevel(logging.WARNING)  # Only show WARNING level and above
+    root_logger.propagate = False
+
+    # Terminal (console) handler
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(logging.WARNING)
+
+    # File handler
+    file_handler = logging.FileHandler(log_filename)
+    file_handler.setLevel(logging.DEBUG)  # Ensure DEBUG level for detailed logs
+
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    console_handler.setFormatter(formatter)
+    file_handler.setFormatter(formatter)
+
+    root_logger.addHandler(console_handler)
+    root_logger.addHandler(file_handler)
+
+    # Ensure colored logs for terminal
+    coloredlogs.install(level=logging.WARNING, logger=root_logger, stream=console_handler.stream)
+
+    # werkzeug logs
+    werkzeug_logger = logging.getLogger('werkzeug')
+    werkzeug_logger.setLevel(logging.INFO)  # Log level to INFO for console
+    
+    # Add logs for werkzeug in terminal
+    werkzeug_console_handler = logging.StreamHandler()
+    werkzeug_console_handler.setLevel(logging.INFO)
+    werkzeug_console_handler.setFormatter(formatter)
+    werkzeug_logger.addHandler(werkzeug_console_handler)
+    werkzeug_logger.propagate = False
+
+    return  init_logger()
+
+
+def init_logger(module_name: str|None=None, level=logging.DEBUG) -> logging.Logger:
+    """"pico.log"
+
+    Args:
+        module_name (str | None, optional): The name of the module for the logger. Defaults to None.
+        level (int, optional): The logging level. Defaults to logging.DEBUG.
+            Levels (from high to low): logging.CRITICAL logging.ERROR
+                                       logging.WARNING logging.INFO logging.DEBUG
+
+    Returns:
+        logging.Logger: The configured logger instance
+    """
+    
+    global log_filename
+                   
+    def get_module_name() -> str | None:
+        cur_frame = inspect.currentframe()
+        if not cur_frame:
+            return None
+        frame = cur_frame.f_back
+        if not frame:
+            return None
+        module = inspect.getmodule(frame)
+        if not module:
+            return None
+        return module.__name__
+
+    if module_name is None:
+        module_name = get_module_name() or "pico"
+            
+    logger = logging.getLogger(module_name)
+    logger.setLevel(level)
+    logger.propagate = False
+    
+    if not logger.handlers:
+        # Terminal (console) handler
+        console_handler = logging.StreamHandler()
+        console_handler.setLevel(level)
+        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levellevel)s - %(message)s')
+        console_handler.setFormatter(formatter)
+        logger.addHandler(console_handler)        
+        
+        # File handler
+        if log_filename is not None:
+            file_handler = logging.FileHandler(log_filename)
+            file_handler.setLevel(level)
+            formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+            file_handler.setFormatter(formatter)
+            
+            logger.addHandler(file_handler)
+
+        # Ensure colored logs for terminal
+        coloredlogs.install(level=level, logger=logger, stream=console_handler.stream)
+
+    return logger