cccpm 0.2.1__py3-none-any.whl

cccpm/__init__.py

@@ -0,0 +1 @@
+from cccpm.cpm_analysis import CPMRegression

cccpm/cpm_analysis.py

@@ -0,0 +1,272 @@
+import os
+import logging
+import shutil
+
+from typing import Union, Type
+from tqdm import tqdm
+
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import BaseCrossValidator, BaseShuffleSplit, KFold, RepeatedKFold, StratifiedKFold
+from sklearn.linear_model import LinearRegression
+
+from cccpm.fold import run_inner_folds
+from cccpm.logging import setup_logging
+from cccpm.more_models import BaseCPMModel, LinearCPMModel
+from cccpm.edge_selection import UnivariateEdgeSelection, PThreshold
+from cccpm.results_manager import ResultsManager, PermutationManager
+from cccpm.utils import train_test_split, check_data, impute_missing_values, select_stable_edges, generate_data_insights
+from cccpm.scoring import score_regression_models
+from cccpm.reporting import HTMLReporter
+
+
+class CPMRegression:
+    """
+    This class handles the process of performing CPM Regression with cross-validation and permutation testing.
+    """
+    def __init__(self,
+                 results_directory: str,
+                 cpm_model: Type[BaseCPMModel] = LinearCPMModel,
+                 cv: Union[BaseCrossValidator, BaseShuffleSplit, RepeatedKFold, StratifiedKFold] = KFold(n_splits=10, shuffle=True, random_state=42),
+                 inner_cv: Union[BaseCrossValidator, BaseShuffleSplit, RepeatedKFold, StratifiedKFold] = None,
+                 edge_selection: UnivariateEdgeSelection = UnivariateEdgeSelection(
+                     edge_statistic='pearson',
+                     edge_selection=[PThreshold(threshold=[0.05], correction=[None])]
+                 ),
+                 select_stable_edges: bool = False,
+                 stability_threshold: float = 0.8,
+                 impute_missing_values: bool = True,
+                 calculate_residuals: bool = False,
+                 n_permutations: int = 0,
+                 atlas_labels: str = None):
+        """
+        Initialize the CPMRegression object.
+
+        Parameters
+        ----------
+        results_directory: str
+            Directory to save results.
+        cv: Union[BaseCrossValidator, BaseShuffleSplit]
+            Outer cross-validation strategy.
+        inner_cv: Union[BaseCrossValidator, BaseShuffleSplit]
+            Inner cross-validation strategy for edge selection.
+        edge_selection:  UnivariateEdgeSelection
+            Method for edge selection.
+        impute_missing_values: bool
+            Whether to impute missing values.
+        n_permutations: int
+            Number of permutations to run for permutation testing.
+        atlas_labels: str
+            CSV file containing atlas and regions labels.
+        """
+        self.results_directory = results_directory
+        self.cpm_model = cpm_model
+        self.cv = cv
+        self.inner_cv = inner_cv
+        self.edge_selection = edge_selection
+        self.select_stable_edges = select_stable_edges
+        self.stability_threshold = stability_threshold
+        self.impute_missing_values = impute_missing_values
+        self.calculate_residuals = calculate_residuals
+        self.n_permutations = n_permutations
+
+        np.random.seed(42)
+        os.makedirs(self.results_directory, exist_ok=True)
+        os.makedirs(os.path.join(self.results_directory, "edges"), exist_ok=True)
+        setup_logging(os.path.join(self.results_directory, "cpm_log.txt"))
+        self.logger = logging.getLogger(__name__)
+
+        # Log important configuration details
+        self._log_analysis_details()
+
+        # check inner cv and param grid
+        if self.inner_cv is None:
+            if len(self.edge_selection.param_grid) > 1:
+                raise RuntimeError("Multiple hyperparameter configurations but no inner cv defined. "
+                                   "Please provide only one hyperparameter configuration or an inner cv.")
+            if self.select_stable_edges:
+                raise RuntimeError("Stable edges can only be selected when using an inner cv.")
+
+        # check and copy atlas labels file
+        self.atlas_labels = self._validate_and_copy_atlas_file(atlas_labels)
+
+        # results are saved to the results manager instance
+        self.results_manager = None
+
+    def _log_analysis_details(self):
+        """
+        Log important information about the analysis in a structured format.
+        """
+        self.logger.info("Starting CPM Regression Analysis")
+        self.logger.info("="*50)
+        self.logger.info(f"Results Directory:       {self.results_directory}")
+        self.logger.info(f"CPM Model:               {self.cpm_model.name}")
+        self.logger.info(f"Outer CV strategy:       {self.cv}")
+        self.logger.info(f"Inner CV strategy:       {self.inner_cv}")
+        self.logger.info(f"Edge selection method:   {self.edge_selection}")
+        self.logger.info(f"Select stable edges:     {'Yes' if self.select_stable_edges else 'No'}")
+        if self.select_stable_edges:
+            self.logger.info(f"Stability threshold:     {self.stability_threshold}")
+        self.logger.info(f"Impute Missing Values:   {'Yes' if self.impute_missing_values else 'No'}")
+        self.logger.info(f"Calculate residuals:     {'Yes' if self.calculate_residuals else 'No'}")
+        self.logger.info(f"Number of Permutations:  {self.n_permutations}")
+        self.logger.info("="*50)
+
+    def _validate_and_copy_atlas_file(self, csv_path):
+        """
+        Validates that a CSV file exists and contains the required columns ('x', 'y', 'z', 'region').
+        If valid, copies it to <self.results_directory>/edges.
+        """
+        if csv_path is None:
+            return None
+
+        required_columns = {"x", "y", "z", "region"}
+        csv_path = os.path.abspath(csv_path)
+
+        # Check if file exists
+        if not os.path.isfile(csv_path):
+            raise RuntimeError(f"CSV file does not exist: {csv_path}")
+
+        # Try to read and validate columns
+        try:
+            df = pd.read_csv(csv_path)
+            missing = required_columns - set(df.columns)
+
+            if missing:
+                raise RuntimeError(f"CSV file is missing required columns: {', '.join(missing)}")
+        except Exception as e:
+            raise RuntimeError(f"Error reading CSV file {csv_path}: {e}")
+
+        # File and columns valid, proceed to copy
+        dest_path = os.path.join(self.results_directory, "edges", os.path.basename(csv_path))
+
+        try:
+            shutil.copy(csv_path, dest_path)
+            self.logger.info(f"Copied CSV file to {dest_path}")
+            return dest_path
+        except Exception as e:
+            self.logger.error(f"Error copying file to {dest_path}: {e}")
+            return None
+
+    def run(self,
+            X: Union[pd.DataFrame, np.ndarray],
+            y: Union[pd.Series, pd.DataFrame, np.ndarray],
+            covariates: Union[pd.Series, pd.DataFrame, np.ndarray]):
+        """
+        Estimates a model using the provided data and conducts permutation testing. This method first fits the model to the actual data and subsequently performs estimation on permuted data for a specified number of permutations. Finally, it calculates permutation results.
+
+        Parameters
+        ----------
+        X: Feature data used for the model. Can be a pandas DataFrame or a NumPy array.
+        y: Target variable used in the estimation process. Can be a pandas Series, DataFrame, or a NumPy array.
+        covariates: Additional covariate data to include in the model. Can be a pandas Series, DataFrame, or a NumPy array.
+
+        """
+        self.logger.info(f"Starting estimation with {self.n_permutations} permutations.")
+
+        # check data and convert to numpy
+        generate_data_insights(X=X, y=y, covariates=covariates, results_directory=self.results_directory)
+        X, y, covariates = check_data(X, y, covariates, impute_missings=self.impute_missing_values)
+
+        # Estimate models on actual data
+        self._single_run(X=X, y=y, covariates=covariates, perm_run=0)
+        self.logger.info("=" * 50)
+
+        # Estimate models on permuted data
+        for perm_id in tqdm(range(1, self.n_permutations + 1), desc="Permutation runs", unit="run",
+                            total=self.n_permutations):
+            y = np.random.permutation(y)
+            self._single_run(X=X, y=y, covariates=covariates, perm_run=perm_id)
+
+        if self.n_permutations > 0:
+            PermutationManager.calculate_permutation_results(self.results_directory, self.logger)
+        self.logger.info("Estimation completed.")
+        self.logger.info("Generating results file.")
+        reporter = HTMLReporter(results_directory=self.results_directory, atlas_labels=self.atlas_labels)
+        reporter.generate_html_report()
+
+    def generate_html_report(self):
+        self.logger.info("Generating HTML report.")
+        reporter = HTMLReporter(results_directory=self.results_directory, atlas_labels=self.atlas_labels)
+        reporter.generate_html_report()
+
+    def _single_run(self,
+                    X: Union[pd.DataFrame, np.ndarray],
+                    y: Union[pd.Series, pd.DataFrame, np.ndarray],
+                    covariates: Union[pd.Series, pd.DataFrame, np.ndarray],
+                    perm_run: int = 0):
+        """
+        Perform an estimation run (either real or permuted data). Includes outer cross-validation loop. For permutation
+        runs, the same strategy is used, but printing is less verbose and the results folder changes.
+
+        :param X: Features (predictors).
+        :param y: Labels (target variable).
+        :param covariates: Covariates to control for.
+        :param perm_run: Permutation run identifier.
+        """
+        results_manager = ResultsManager(output_dir=self.results_directory, perm_run=perm_run,
+                                         n_folds=self.cv.get_n_splits(), n_features=X.shape[1])
+
+        iterator = (
+            tqdm(
+                enumerate(self.cv.split(X, y)),
+                total=self.cv.get_n_splits(),
+                desc="Running outer folds",
+                unit="fold"
+            )
+            if not perm_run else
+            enumerate(self.cv.split(X, y))
+        )
+        for outer_fold, (train, test) in iterator:
+            # split according to single outer fold
+            X_train, X_test, y_train, y_test, cov_train, cov_test = train_test_split(train, test, X, y, covariates)
+
+            # impute missing values
+            if self.impute_missing_values:
+                X_train, X_test, cov_train, cov_test = impute_missing_values(X_train, X_test, cov_train, cov_test)
+
+            # residualize X to remove effect of covariates
+            if self.calculate_residuals:
+                residual_model = LinearRegression().fit(cov_train, X_train)
+                X_train = X_train - residual_model.predict(cov_train)
+                X_test = X_test - residual_model.predict(cov_test)
+
+            # if the user specified an inner cross-validation, estimate models witin inner loop
+            if self.inner_cv:
+                best_params, stability_edges = run_inner_folds(cpm_model=self.cpm_model,
+                                                               X=X_train, y=y_train, covariates=cov_train,
+                                                               inner_cv=self.inner_cv,
+                                                               edge_selection=self.edge_selection,
+                                                               results_directory=os.path.join(results_manager.results_directory, 'folds', str(outer_fold)),
+                                                               perm_run=perm_run)
+            else:
+                best_params = self.edge_selection.param_grid[0]
+
+            # Use best parameters to estimate performance on outer fold test set
+            if self.select_stable_edges:
+                edges = select_stable_edges(stability_edges, self.stability_threshold)
+            else:
+                self.edge_selection.set_params(**best_params)
+                edges = self.edge_selection.fit_transform(X=X_train, y=y_train, covariates=cov_train).return_selected_edges()
+
+            results_manager.store_edges(edges=edges, fold=outer_fold)
+
+            # Build model and make predictions
+            model = self.cpm_model(edges=edges).fit(X_train, y_train, cov_train)
+            y_pred = model.predict(X_test, cov_test)
+            network_strengths = model.get_network_strengths(X_test, cov_test)
+            metrics = score_regression_models(y_true=y_test, y_pred=y_pred)
+            results_manager.store_predictions(y_pred=y_pred, y_true=y_test, params=best_params, fold=outer_fold,
+                                              param_id=0, test_indices=test)
+            results_manager.store_metrics(metrics=metrics, params=best_params, fold=outer_fold, param_id=0)
+            results_manager.store_network_strengths(network_strengths=network_strengths, y_true=y_test, fold=outer_fold)
+
+        # once all outer folds are done, calculate final results and edge stability
+        results_manager.calculate_final_cv_results()
+        results_manager.calculate_edge_stability()
+
+        if not perm_run:
+            self.logger.info(results_manager.agg_results.round(4).to_string())
+            results_manager.save_predictions()
+            results_manager.save_network_strengths()
+            self.results_manager = results_manager

cccpm/edge_selection.py

@@ -0,0 +1,271 @@
+import pandas as pd
+import numpy as np
+from scipy.stats import ttest_1samp, t, rankdata
+from typing import Union
+
+from sklearn.base import BaseEstimator
+from sklearn.model_selection import ParameterGrid
+import statsmodels.stats.multitest as multitest
+from warnings import filterwarnings
+
+
+def one_sample_t_test(matrix, population_mean):
+    # Calculate the mean and standard deviation along the rows
+    sample_means = np.mean(matrix, axis=0)
+    sample_stds = np.std(matrix, axis=0, ddof=1)
+    n = matrix.shape[1]  # Number of samples in each row
+
+    # Calculate the t-statistics
+    filterwarnings('ignore', category=RuntimeWarning)
+    t_stats = (sample_means - population_mean) / (sample_stds / np.sqrt(n))
+
+    # Calculate the p-values using the t-distribution survival function
+    p_values = 2 * t.sf(np.abs(t_stats), df=n - 1)
+
+    return t_stats, p_values
+
+
+def compute_t_and_p_values(correlations, df):
+    # Calculate t-statistics
+    t_stats = correlations * np.sqrt(df / (1 - correlations ** 2))
+    # Calculate p-values
+    p_values = 2 * t.sf(np.abs(t_stats), df=df)
+    return t_stats, p_values
+
+
+def compute_correlation_and_pvalues(x, Y, rank=False):
+    n = len(x)
+
+    if rank:
+        x = rankdata(x)
+        Y = rankdata(Y, axis=0)
+
+    # Mean-centering
+    x_centered = x - np.mean(x)
+    Y_centered = Y - np.mean(Y, axis=0)
+
+    # Correlation calculation
+    corr_numerator = np.dot(Y_centered.T, x_centered)
+    corr_denominator = (np.sqrt(np.sum(Y_centered ** 2, axis=0)) * np.sqrt(np.sum(x_centered ** 2)))
+
+    correlations = corr_numerator / corr_denominator
+
+    # Calculate t-statistics and p-values
+    _, p_values = compute_t_and_p_values(correlations, n - 2)
+
+    return correlations, p_values
+
+
+def get_residuals(X, Z):
+    # Add a column of ones to Z for the intercept
+    Z = np.hstack([Z, np.ones((Z.shape[0], 1))])
+
+    # Compute the coefficients using the normal equation
+    B = np.linalg.lstsq(Z, X, rcond=None)[0]
+
+    # Predict X from Z
+    X_hat = Z.dot(B)
+
+    # Compute residuals
+    residuals = X - X_hat
+
+    return residuals
+
+
+def semi_partial_correlation(x, Y, Z, rank=False):
+    # ToDo: THIS IS A PARTIAL CORRELATION, NOT SEMI-PARTIAL
+    if rank:
+        x = rankdata(x)
+        Y = rankdata(Y, axis=0)
+        Z = rankdata(Z, axis=0)
+
+        #Y = np.apply_along_axis(rankdata, 0, Y)
+        #Z = np.apply_along_axis(rankdata, 0, Z)
+
+    # Calculate residuals for x and each column in Y
+    x_residuals = get_residuals(x.reshape(-1, 1), Z).ravel()
+    Y_residuals = get_residuals(Y, Z)
+
+    # Mean-centering the residuals
+    x_centered = x_residuals - np.mean(x_residuals)
+    Y_centered = Y_residuals - np.mean(Y_residuals, axis=0)
+
+    # Correlation calculation
+    corr_numerator = np.dot(Y_centered.T, x_centered)
+    corr_denominator = (np.sqrt(np.sum(Y_centered ** 2, axis=0)) * np.sqrt(np.sum(x_centered ** 2)))
+    partial_corr = corr_numerator / corr_denominator
+
+    # Calculate t-statistics and p-values
+    n = len(x)
+    k = Z.shape[1]
+    _, p_values = compute_t_and_p_values(partial_corr, n - k - 2)
+
+    return partial_corr, p_values
+
+
+def pearson_correlation_with_pvalues(x, Y):
+    return compute_correlation_and_pvalues(x, Y, rank=False)
+
+
+def spearman_correlation_with_pvalues(x, Y):
+    return compute_correlation_and_pvalues(x, Y, rank=True)
+
+
+def semi_partial_correlation_pearson(x, Y, Z):
+    return semi_partial_correlation(x, Y, Z, rank=False)
+
+
+def semi_partial_correlation_spearman(x, Y, Z):
+    return semi_partial_correlation(x, Y, Z, rank=True)
+
+
+class BaseEdgeSelector(BaseEstimator):
+    def select(self, r, p):
+        pass
+
+
+class PThreshold(BaseEdgeSelector):
+    def __init__(self, threshold: Union[float, list] = 0.05, correction: Union[str, list] = None):
+        """
+
+        :param threshold:
+        :param correction: can be one of statsmodels methods
+                            bonferroni : one-step correction
+                            sidak : one-step correction
+                            holm-sidak : step down method using Sidak adjustments
+                            holm : step-down method using Bonferroni adjustments
+                            simes-hochberg : step-up method (independent)
+                            hommel : closed method based on Simes tests (non-negative)
+                            fdr_bh : Benjamini/Hochberg (non-negative)
+                            fdr_by : Benjamini/Yekutieli (negative)
+                            fdr_tsbh : two stage fdr correction (non-negative)
+                            fdr_tsbky : two stage fdr correction (non-negative)
+        """
+        self._threshold = None
+        self._correction = None
+        self.threshold = threshold
+        self.correction = correction
+
+    @property
+    def threshold(self):
+        if isinstance(self._threshold, (int, float)):
+            return [float(self._threshold)]
+        return self._threshold or [0.05]
+
+    @threshold.setter
+    def threshold(self, value):
+        if isinstance(value, (int, float)):
+            self._threshold = float(value)
+        elif isinstance(value, list):
+            self._threshold = value
+        else:
+            raise ValueError("threshold must be float or list")
+
+    @property
+    def correction(self):
+        if self._correction is None:
+            return [None]
+        if isinstance(self._correction, str):
+            return [self._correction]
+        return self._correction
+
+    @correction.setter
+    def correction(self, value):
+        if value is None:
+            self._correction = None
+        elif isinstance(value, str):
+            self._correction = value
+        elif isinstance(value, list):
+            self._correction = value
+        else:
+            raise ValueError("correction must be None, str, or list")
+
+    def select(self, r, p):
+        if self._correction is not None:
+            _, p, _, _ = multitest.multipletests(p, alpha=0.05, method=self._correction)
+        pos_edges = np.where((p < self.threshold) & (r > 0))[0]
+        neg_edges = np.where((p < self.threshold) & (r < 0))[0]
+        return {'positive': pos_edges, 'negative': neg_edges}
+
+
+class SelectPercentile(BaseEdgeSelector):
+    def __init__(self, percentile: Union[float, list] = 0.05):
+        self.percentile = percentile
+
+
+class SelectKBest(BaseEdgeSelector):
+    def __init__(self, k: Union[int, list] = None):
+        self.k = k
+
+
+class EdgeStatistic(BaseEstimator):
+    def __init__(self, edge_statistic: str = 'spearman', t_test_filter: bool = False):
+        self.edge_statistic = edge_statistic
+        self.t_test_filter = t_test_filter
+
+    def fit_transform(self,
+                      X: Union[pd.DataFrame, np.ndarray],
+                      y: Union[pd.Series, pd.DataFrame, np.ndarray],
+                      covariates: Union[pd.Series, pd.DataFrame, np.ndarray]):
+        r_edges, p_edges = np.zeros(X.shape[1]), np.ones(X.shape[1])
+        #if self.t_test_filter:
+        #    _, p_values = one_sample_t_test(X, 0)
+        #    valid_edges = p_values < 0.05
+        #else:
+        #    valid_edges = np.bool(np.ones(X.shape[1]))
+
+        from sklearn.feature_selection import VarianceThreshold
+        selector = VarianceThreshold(threshold=0.01)
+        selector.fit(X)
+        valid_edges = selector.get_support()
+
+
+        if self.edge_statistic == 'pearson':
+            r_edges_masked, p_edges_masked = pearson_correlation_with_pvalues(y, X[:, valid_edges])
+        elif self.edge_statistic == 'spearman':
+            r_edges_masked, p_edges_masked = spearman_correlation_with_pvalues(y, X[:, valid_edges])
+        elif self.edge_statistic == 'pearson_partial':
+            r_edges_masked, p_edges_masked = semi_partial_correlation_pearson(y, X[:, valid_edges], covariates)
+        elif self.edge_statistic == 'spearman_partial':
+            r_edges_masked, p_edges_masked = semi_partial_correlation_spearman(y, X[:, valid_edges], covariates)
+        else:
+            raise NotImplemented("Unsupported edge selection method")
+        r_edges[valid_edges] = r_edges_masked
+        p_edges[valid_edges] = p_edges_masked
+        return r_edges, p_edges
+
+
+class UnivariateEdgeSelection(BaseEstimator):
+    def __init__(self,
+                 edge_statistic: str = 'spearman',
+                 t_test_filter: bool = False,
+                 edge_selection: Union[list, None, PThreshold] = None,
+                 ):
+        self.r_edges = None
+        self.p_edges = None
+        self.t_test_filter = t_test_filter
+        self.edge_statistic = EdgeStatistic(edge_statistic=edge_statistic, t_test_filter=t_test_filter)
+        self.edge_selection = edge_selection
+        if isinstance(edge_selection, (list, tuple)):
+            self.edge_selection = edge_selection
+        else:
+            self.edge_selection = [edge_selection]
+        self.param_grid = self._generate_config_grid()
+
+    def _generate_config_grid(self):
+        grid_elements = []
+        for selector in self.edge_selection:
+            params = {}
+            params['edge_selection'] = [selector]
+            for key, value in selector.get_params().items():
+                params['edge_selection__' + key] = value
+            grid_elements.append(params)
+        return ParameterGrid(grid_elements)
+
+    def fit_transform(self, X, y=None, covariates=None):
+        self.r_edges, self.p_edges = self.edge_statistic.fit_transform(X=X, y=y, covariates=covariates)
+        return self
+
+    def return_selected_edges(self):
+        selected_edges = self.edge_selection.select(r=self.r_edges, p=self.p_edges)
+        return selected_edges

cccpm/fold.py

@@ -0,0 +1,46 @@
+from cccpm.utils import train_test_split
+from cccpm.scoring import score_regression_models
+from cccpm.results_manager import ResultsManager
+from cccpm.edge_selection import BaseEdgeSelector
+
+
+def run_inner_folds(cpm_model, X, y, covariates, inner_cv, edge_selection: BaseEdgeSelector, results_directory,
+                    perm_run):
+    """
+    Run inner cross-validation over all folds and hyperparameter configurations.
+
+    Returns
+    -------
+    cv_results : DataFrame
+        Aggregated results from all inner folds.
+    stability_edges : dict
+        Dictionary with 'positive' and 'negative' keys mapping to arrays of edge stability scores.
+    """
+    param_grid = edge_selection.param_grid
+    n_features = X.shape[1]
+    n_params = len(param_grid)
+    n_folds = inner_cv.get_n_splits()
+
+    results_manager = ResultsManager(output_dir=results_directory, perm_run=perm_run,
+                                     n_folds=n_folds, n_features=n_features, n_params=n_params)
+
+    for fold_id, (train, test) in enumerate(inner_cv.split(X, y)):
+        # split according to single fold
+        X_train, X_test, y_train, y_test, cov_train, cov_test = train_test_split(train, test, X, y, covariates)
+
+        for param_id, config in enumerate(param_grid):
+            edge_selection.set_params(**config)
+            selected_edges = edge_selection.fit_transform(X_train, y_train, cov_train).return_selected_edges()
+            y_pred = cpm_model(edges=selected_edges).fit(X_train, y_train, cov_train).predict(X_test, cov_test)
+            metrics = score_regression_models(y_true=y_test, y_pred=y_pred)
+
+            results_manager.store_edges(selected_edges, fold_id, param_id)
+            results_manager.store_metrics(metrics=metrics, params=config, fold=fold_id, param_id=param_id)
+
+    # once all outer folds are done, calculate final results and edge stability
+    results_manager.aggregate_inner_folds()
+
+    best_params, best_param_id = results_manager.find_best_params()
+    stability_edges = results_manager.calculate_edge_stability(write=False, best_param_id=best_param_id)
+
+    return best_params, stability_edges

cccpm/logging.py

@@ -0,0 +1,37 @@
+import sys
+import logging
+
+
+def setup_logging(log_file: str = "analysis_log.txt"):
+    # Get the root logger
+    logger = logging.getLogger()
+
+    # Check if handlers already exist and remove them to avoid duplication
+    if logger.hasHandlers():
+        logger.handlers.clear()
+
+    # Console handler: logs all levels (DEBUG and above) to the console
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setLevel(logging.DEBUG)
+    console_handler.setFormatter(SimpleFormatter())
+
+    # File handler: logs only INFO level logs to the file
+    file_handler = logging.FileHandler(log_file, mode='w')
+    file_handler.setLevel(logging.INFO)
+    file_handler.addFilter(lambda record: record.levelno == logging.INFO)
+    file_handler.setFormatter(SimpleFormatter())
+
+    # Create a logger and set the base level to DEBUG so both handlers can operate independently
+    logger.setLevel(logging.DEBUG)  # This ensures all messages are passed to handlers
+    logger.addHandler(console_handler)
+    logger.addHandler(file_handler)
+
+    logging.getLogger('matplotlib').setLevel(logging.WARNING)
+    logging.getLogger('matplotlib.font_manager').setLevel(logging.WARNING)
+
+
+class SimpleFormatter(logging.Formatter):
+    def format(self, record):
+        log_fmt = "%(message)s"
+        formatter = logging.Formatter(log_fmt)
+        return formatter.format(record)