spacr 0.3.81__py3-none-any.whl → 0.4.1__py3-none-any.whl

spacr/sp_stats.py

@@ -0,0 +1,221 @@
+from scipy.stats import shapiro, normaltest, levene, ttest_ind, mannwhitneyu, kruskal, f_oneway
+from statsmodels.stats.multicomp import pairwise_tukeyhsd
+import scikit_posthocs as sp
+import numpy as np
+import pandas as pd
+from scipy.stats import chi2_contingency, fisher_exact
+import itertools
+from statsmodels.stats.multitest import multipletests
+
+
+def choose_p_adjust_method(num_groups, num_data_points):
+    """
+    Selects the most appropriate p-value adjustment method based on data characteristics.
+    
+    Parameters:
+    - num_groups: Number of unique groups being compared
+    - num_data_points: Number of data points per group (assuming balanced groups)
+    
+    Returns:
+    - A string representing the recommended p-adjustment method
+    """
+    num_comparisons = (num_groups * (num_groups - 1)) // 2  # Number of pairwise comparisons
+
+    # Decision logic for choosing the adjustment method
+    if num_comparisons <= 10 and num_data_points > 5:
+        return 'holm'  # Balanced between power and Type I error control
+    elif num_comparisons > 10 and num_data_points <= 5:
+        return 'fdr_bh'  # FDR control for large number of comparisons and small sample size
+    elif num_comparisons <= 10:
+        return 'sidak'  # Less conservative than Bonferroni, good for independent comparisons
+    else:
+        return 'bonferroni'  # Very conservative, use for strict control of Type I errors
+
+def perform_normality_tests(df, grouping_column, data_columns):
+    """Perform normality tests for each group and data column."""
+    unique_groups = df[grouping_column].unique()
+    normality_results = []
+
+    for column in data_columns:
+        for group in unique_groups:
+            data = df.loc[df[grouping_column] == group, column].dropna()
+            n_samples = len(data)
+
+            if n_samples < 3:
+                # Skip test if there aren't enough data points
+                print(f"Skipping normality test for group '{group}' on column '{column}' - Not enough data.")
+                normality_results.append({
+                    'Comparison': f'Normality test for {group} on {column}',
+                    'Test Statistic': None,
+                    'p-value': None,
+                    'Test Name': 'Skipped',
+                    'Column': column,
+                    'n': n_samples
+                })
+                continue
+
+            # Choose the appropriate normality test based on the sample size
+            if n_samples >= 8:
+                stat, p_value = normaltest(data)
+                test_name = "D'Agostino-Pearson test"
+            else:
+                stat, p_value = shapiro(data)
+                test_name = "Shapiro-Wilk test"
+
+            normality_results.append({
+                'Comparison': f'Normality test for {group} on {column}',
+                'Test Statistic': stat,
+                'p-value': p_value,
+                'Test Name': test_name,
+                'Column': column,
+                'n': n_samples
+            })
+
+        # Check if all groups are normally distributed (p > 0.05)
+        normal_p_values = [result['p-value'] for result in normality_results if result['Column'] == column and result['p-value'] is not None]
+        is_normal = all(p > 0.05 for p in normal_p_values)
+
+    return is_normal, normality_results
+
+
+def perform_levene_test(df, grouping_column, data_column):
+    """Perform Levene's test for equal variance."""
+    unique_groups = df[grouping_column].unique()
+    grouped_data = [df.loc[df[grouping_column] == group, data_column].dropna() for group in unique_groups]
+    stat, p_value = levene(*grouped_data)
+    return stat, p_value
+
+def perform_statistical_tests(df, grouping_column, data_columns, paired=False):
+    """Perform statistical tests for each data column."""
+    unique_groups = df[grouping_column].unique()
+    test_results = []
+
+    for column in data_columns:
+        grouped_data = [df.loc[df[grouping_column] == group, column].dropna() for group in unique_groups]
+        if len(unique_groups) == 2:  # For two groups
+            if paired:
+                print("Performing paired tests (not implemented in this template).")
+                continue  # Extend as needed
+            else:
+                # Check normality for two groups
+                is_normal, _ = perform_normality_tests(df, grouping_column, [column])
+                if is_normal:
+                    stat, p = ttest_ind(grouped_data[0], grouped_data[1])
+                    test_name = 'T-test'
+                else:
+                    stat, p = mannwhitneyu(grouped_data[0], grouped_data[1])
+                    test_name = 'Mann-Whitney U test'
+        else:
+            # Check normality for multiple groups
+            is_normal, _ = perform_normality_tests(df, grouping_column, [column])
+            if is_normal:
+                stat, p = f_oneway(*grouped_data)
+                test_name = 'One-way ANOVA'
+            else:
+                stat, p = kruskal(*grouped_data)
+                test_name = 'Kruskal-Wallis test'
+
+        test_results.append({
+            'Column': column,
+            'Test Name': test_name,
+            'Test Statistic': stat,
+            'p-value': p,
+            'Groups': len(unique_groups)
+        })
+
+    return test_results
+
+
+def perform_posthoc_tests(df, grouping_column, data_column, is_normal):
+    """Perform post-hoc tests for multiple groups with both original and adjusted p-values."""
+    unique_groups = df[grouping_column].unique()
+    posthoc_results = []
+
+    if len(unique_groups) > 2:
+        num_groups = len(unique_groups)
+        num_data_points = len(df[data_column].dropna()) // num_groups  # Assuming roughly equal data points per group
+        p_adjust_method = choose_p_adjust_method(num_groups, num_data_points)
+
+        if is_normal:
+            # Tukey's HSD automatically adjusts p-values
+            tukey_result = pairwise_tukeyhsd(df[data_column], df[grouping_column], alpha=0.05)
+            for comparison, p_value in zip(tukey_result._results_table.data[1:], tukey_result.pvalues):
+                posthoc_results.append({
+                    'Comparison': f"{comparison[0]} vs {comparison[1]}",
+                    'Original p-value': None,  # Tukey HSD does not provide raw p-values
+                    'Adjusted p-value': p_value,
+                    'Adjusted Method': 'Tukey HSD',
+                    'Test Name': 'Tukey HSD'
+                })
+        else:
+            # Dunn's test with p-value adjustment
+            raw_dunn_result = sp.posthoc_dunn(df, val_col=data_column, group_col=grouping_column, p_adjust=None)
+            adjusted_dunn_result = sp.posthoc_dunn(df, val_col=data_column, group_col=grouping_column, p_adjust=p_adjust_method)
+            for i, group_a in enumerate(adjusted_dunn_result.index):
+                for j, group_b in enumerate(adjusted_dunn_result.columns):
+                    if i < j:  # Only consider unique pairs
+                        posthoc_results.append({
+                            'Comparison': f"{group_a} vs {group_b}",
+                            'Original p-value': raw_dunn_result.iloc[i, j],
+                            'Adjusted p-value': adjusted_dunn_result.iloc[i, j],
+                            'Adjusted Method': p_adjust_method,
+                            'Test Name': "Dunn's Post-hoc"
+                        })
+
+    return posthoc_results
+
+def chi_pairwise(raw_counts, verbose=False):
+    """
+    Perform pairwise chi-square or Fisher's exact tests between all unique group pairs
+    and apply p-value correction.
+
+    Parameters:
+    - raw_counts (DataFrame): Contingency table with group-wise counts.
+    - verbose (bool): Whether to print results for each pair.
+
+    Returns:
+    - pairwise_df (DataFrame): DataFrame with pairwise test results, including corrected p-values.
+    """
+    pairwise_results = []
+    groups = raw_counts.index.unique()  # Use index from raw_counts for group pairs
+    raw_p_values = []  # Store raw p-values for correction later
+    
+    # Calculate the number of groups and average number of data points per group
+    num_groups = len(groups)
+    num_data_points = raw_counts.sum(axis=1).mean()  # Average total data points per group
+    p_adjust_method = choose_p_adjust_method(num_groups, num_data_points)
+
+    for group1, group2 in itertools.combinations(groups, 2):
+        contingency_table = raw_counts.loc[[group1, group2]].values
+        if contingency_table.shape[1] == 2:  # Fisher's Exact Test for 2x2 tables
+            oddsratio, p_value = fisher_exact(contingency_table)
+            test_name = "Fisher's Exact Test"
+        else:  # Chi-Square Test for larger tables
+            chi2_stat, p_value, _, _ = chi2_contingency(contingency_table)
+            test_name = 'Pairwise Chi-Square Test'
+        
+        pairwise_results.append({
+            'Group 1': group1,
+            'Group 2': group2,
+            'Test Name': test_name,
+            'p-value': p_value
+        })
+        raw_p_values.append(p_value)
+
+    # Apply p-value correction
+    corrected_p_values = multipletests(raw_p_values, method=p_adjust_method)[1]
+
+    # Add corrected p-values to results
+    for i, result in enumerate(pairwise_results):
+        result['p-value_adj'] = corrected_p_values[i]
+
+    pairwise_df = pd.DataFrame(pairwise_results)
+    
+    pairwise_df['adj'] = p_adjust_method
+
+    if verbose:
+        # Print pairwise results
+        print("\nPairwise Frequency Analysis Results:")
+        print(pairwise_df.to_string(index=False))
+
+    return pairwise_df

spacr/submodules.py

@@ -1,3 +1,6 @@
+
+
+
 import seaborn as sns
 import os, random, sqlite3, re, shap
 import pandas as pd
@@ -10,7 +13,10 @@ from IPython.display import display
 from sklearn.ensemble import RandomForestClassifier
 from sklearn.inspection import permutation_importance
 from math import pi
-from scipy.stats import chi2_contingency
+from scipy.stats import chi2_contingency, pearsonr
+from scipy.spatial.distance import cosine
+
+from sklearn.metrics import mean_absolute_error
 
 import matplotlib.pyplot as plt
 from natsort import natsorted
@@ -1035,7 +1041,7 @@ def analyze_class_proportion(settings):
     from .io import _read_and_merge_data
     from .settings import set_analyze_class_proportion_defaults
     from .plot import plot_plates, plot_proportion_stacked_bars
-    from .stats import perform_normality_tests, perform_levene_test, perform_statistical_tests, perform_posthoc_tests
+    from .sp_stats import perform_normality_tests, perform_levene_test, perform_statistical_tests, perform_posthoc_tests
     
     settings = set_analyze_class_proportion_defaults(settings)
     save_settings(settings, name='analyze_class_proportion', show=True)
@@ -1132,3 +1138,278 @@ def analyze_class_proportion(settings):
         print("Statistical analysis results saved.")
 
     return output
+
+def generate_score_heatmap(settings):
+    
+    def group_cv_score(csv, plate=1, column='c3', data_column='pred'):
+        
+        df = pd.read_csv(csv)
+        if 'col' in df.columns:
+            df = df[df['col']==column]
+        elif 'column' in df.columns:
+            df['col'] = df['column']
+            df = df[df['col']==column]
+        if not plate is None:
+            df['plate'] = f"plate{plate}"
+        grouped_df = df.groupby(['plate', 'row', 'col'])[data_column].mean().reset_index()
+        grouped_df['prc'] = grouped_df['plate'].astype(str) + '_' + grouped_df['row'].astype(str) + '_' + grouped_df['col'].astype(str)
+        return grouped_df
+
+    def calculate_fraction_mixed_condition(csv, plate=1, column='c3', control_sgrnas = ['TGGT1_220950_1', 'TGGT1_233460_4']):
+        df = pd.read_csv(csv)  
+        df = df[df['column_name']==column]
+        if plate not in df.columns:
+            df['plate'] = f"plate{plate}"
+        df = df[df['grna_name'].str.match(f'^{control_sgrnas[0]}$|^{control_sgrnas[1]}$')]
+        grouped_df = df.groupby(['plate', 'row_name', 'column_name'])['count'].sum().reset_index()
+        grouped_df = grouped_df.rename(columns={'count': 'total_count'})
+        merged_df = pd.merge(df, grouped_df, on=['plate', 'row_name', 'column_name'])
+        merged_df['fraction'] = merged_df['count'] / merged_df['total_count']
+        merged_df['prc'] = merged_df['plate'].astype(str) + '_' + merged_df['row_name'].astype(str) + '_' + merged_df['column_name'].astype(str)
+        return merged_df
+
+    def plot_multi_channel_heatmap(df, column='c3', cmap='coolwarm'):
+        """
+        Plot a heatmap with multiple channels as columns.
+
+        Parameters:
+        - df: DataFrame with scores for different channels.
+        - column: Column to filter by (default is 'c3').
+        """
+        # Extract row number and convert to integer for sorting
+        df['row_num'] = df['row'].str.extract(r'(\d+)').astype(int)
+
+        # Filter and sort by plate, row, and column
+        df = df[df['col'] == column]
+        df = df.sort_values(by=['plate', 'row_num', 'col'])
+
+        # Drop temporary 'row_num' column after sorting
+        df = df.drop('row_num', axis=1)
+
+        # Create a new column combining plate, row, and column for the index
+        df['plate_row_col'] = df['plate'] + '-' + df['row'] + '-' + df['col']
+
+        # Set 'plate_row_col' as the index
+        df.set_index('plate_row_col', inplace=True)
+
+        # Extract only numeric data for the heatmap
+        heatmap_data = df.select_dtypes(include=[float, int])
+
+        # Plot heatmap with square boxes, no annotations, and 'viridis' colormap
+        plt.figure(figsize=(12, 8))
+        sns.heatmap(
+            heatmap_data,
+            cmap=cmap,
+            cbar=True,
+            square=True,
+            annot=False
+        )
+
+        plt.title("Heatmap of Prediction Scores for All Channels")
+        plt.xlabel("Channels")
+        plt.ylabel("Plate-Row-Column")
+        plt.tight_layout()
+
+        # Save the figure object and return it
+        fig = plt.gcf()
+        plt.show()
+
+        return fig
+
+
+    def combine_classification_scores(folders, csv_name, data_column, plate=1, column='c3'):
+        # Ensure `folders` is a list
+        if isinstance(folders, str):
+            folders = [folders]
+
+        ls = []  # Initialize ls to store found CSV file paths
+
+        # Iterate over the provided folders
+        for folder in folders:
+            sub_folders = os.listdir(folder)  # Get sub-folder list
+            for sub_folder in sub_folders:  # Iterate through sub-folders
+                path = os.path.join(folder, sub_folder)  # Join the full path
+
+                if os.path.isdir(path):  # Check if it’s a directory
+                    csv = os.path.join(path, csv_name)  # Join path to the CSV file
+                    if os.path.exists(csv):  # If CSV exists, add to list
+                        ls.append(csv)
+                    else:
+                        print(f'No such file: {csv}')
+
+        # Initialize combined DataFrame
+        combined_df = None
+        print(f'Found {len(ls)} CSV files')
+
+        # Loop through all collected CSV files and process them
+        for csv_file in ls:
+            df = pd.read_csv(csv_file)  # Read CSV into DataFrame
+            df = df[df['col']==column]
+            if not plate is None:
+                df['plate'] = f"plate{plate}"
+            # Group the data by 'plate', 'row', and 'col'
+            grouped_df = df.groupby(['plate', 'row', 'col'])[data_column].mean().reset_index()
+            # Use the CSV filename to create a new column name
+            folder_name = os.path.dirname(csv_file).replace(".csv", "")
+            new_column_name = os.path.basename(f"{folder_name}_{data_column}")
+            print(new_column_name)
+            grouped_df = grouped_df.rename(columns={data_column: new_column_name})
+
+            # Merge into the combined DataFrame
+            if combined_df is None:
+                combined_df = grouped_df
+            else:
+                combined_df = pd.merge(combined_df, grouped_df, on=['plate', 'row', 'col'], how='outer')
+        combined_df['prc'] = combined_df['plate'].astype(str) + '_' + combined_df['row'].astype(str) + '_' + combined_df['col'].astype(str)
+        return combined_df
+    
+    def calculate_mae(df):
+        """
+        Calculate the MAE between each channel's predictions and the fraction column for all rows.
+        """
+        # Extract numeric columns excluding 'fraction' and 'prc'
+        channels = df.drop(columns=['fraction', 'prc']).select_dtypes(include=[float, int])
+
+        mae_data = []
+
+        # Compute MAE for each channel with 'fraction' for all rows
+        for column in channels.columns:
+            for index, row in df.iterrows():
+                mae = mean_absolute_error([row['fraction']], [row[column]])
+                mae_data.append({'Channel': column, 'MAE': mae, 'Row': row['prc']})
+
+        # Convert the list of dictionaries to a DataFrame
+        mae_df = pd.DataFrame(mae_data)
+        return mae_df
+
+    result_df = combine_classification_scores(settings['folders'], settings['csv_name'], settings['data_column'], settings['plate'], settings['column'], )
+    df = calculate_fraction_mixed_condition(settings['csv'], settings['plate'], settings['column'], settings['control_sgrnas'])
+    df = df[df['grna_name']==settings['fraction_grna']]
+    fraction_df = df[['fraction', 'prc']]
+    merged_df = pd.merge(fraction_df, result_df, on=['prc'])
+    cv_df = group_cv_score(settings['cv_csv'], settings['plate'], settings['column'], settings['data_column_cv'])
+    cv_df = cv_df[[settings['data_column_cv'], 'prc']]
+    merged_df = pd.merge(merged_df, cv_df, on=['prc'])
+    
+    fig = plot_multi_channel_heatmap(merged_df, settings['column'], settings['cmap'])
+    if 'row_number' in merged_df.columns:
+        merged_df = merged_df.drop('row_num', axis=1)
+    mae_df = calculate_mae(merged_df)
+    if 'row_number' in mae_df.columns:
+        mae_df = mae_df.drop('row_num', axis=1)
+        
+    if not settings['dst'] is None:
+        mae_dst = os.path.join(settings['dst'], f"mae_scores_comparison_plate_{settings['plate']}.csv")
+        merged_dst = os.path.join(settings['dst'], f"scores_comparison_plate_{settings['plate']}_data.csv")
+        heatmap_save = os.path.join(settings['dst'], f"scores_comparison_plate_{settings['plate']}.pdf")
+        mae_df.to_csv(mae_dst, index=False)
+        merged_df.to_csv(merged_dst, index=False)
+        fig.savefig(heatmap_save, format='pdf', dpi=600, bbox_inches='tight')
+    return merged_df
+
+def post_regression_analysis(csv_file, grna_dict, grna_list, save=False):
+    
+    def _analyze_and_visualize_grna_correlation(df, grna_list, save_folder, save=False):
+        """
+        Analyze and visualize the correlation matrix of gRNAs based on their fractions and overlap.
+
+        Parameters:
+        df (pd.DataFrame): DataFrame with columns ['grna', 'fraction', 'prc'].
+        grna_list (list): List of gRNAs to include in the correlation analysis.
+        save_folder (str): Path to the folder where figures and data will be saved.
+
+        Returns:
+        pd.DataFrame: Correlation matrix of the gRNAs.
+        """
+        # Filter the DataFrame to include only rows with gRNAs in the list
+        filtered_df = df[df['grna'].isin(grna_list)]
+
+        # Pivot the data to create a prc-by-gRNA matrix, using fractions as values
+        pivot_df = filtered_df.pivot_table(index='prc', columns='grna', values='fraction', aggfunc='sum').fillna(0)
+
+        # Compute the correlation matrix
+        correlation_matrix = pivot_df.corr()
+        
+        if save:
+            # Save the correlation matrix
+            correlation_matrix.to_csv(os.path.join(save_folder, 'correlation_matrix.csv'))
+        
+        # Visualize the correlation matrix as a heatmap
+        plt.figure(figsize=(10, 8))
+        sns.heatmap(correlation_matrix, annot=False, cmap='coolwarm', cbar=True)
+        plt.title('gRNA Correlation Matrix')
+        plt.xlabel('gRNAs')
+        plt.ylabel('gRNAs')
+        plt.tight_layout()
+        
+        if save:
+            correlation_fig_path = os.path.join(save_folder, 'correlation_matrix_heatmap.pdf')
+            plt.savefig(correlation_fig_path, dpi=300)
+        
+        plt.show()
+
+        return correlation_matrix
+
+    def _compute_effect_sizes(correlation_matrix, grna_dict, save_folder, save=False):
+        """
+        Compute and visualize the effect sizes of gRNAs given fixed effect sizes for a subset of gRNAs.
+
+        Parameters:
+        correlation_matrix (pd.DataFrame): Correlation matrix of gRNAs.
+        grna_dict (dict): Dictionary of gRNAs with fixed effect sizes {grna_name: effect_size}.
+        save_folder (str): Path to the folder where figures and data will be saved.
+
+        Returns:
+        pd.Series: Effect sizes of all gRNAs.
+        """
+        # Ensure the matrix is symmetric and normalize values to 0-1
+        corr_matrix = correlation_matrix.copy()
+        corr_matrix = (corr_matrix - corr_matrix.min().min()) / (corr_matrix.max().max() - corr_matrix.min().min())
+
+        # Initialize the effect sizes with dtype float
+        effect_sizes = pd.Series(0.0, index=corr_matrix.index)
+
+        # Set the effect sizes for the specified gRNAs
+        for grna, size in grna_dict.items():
+            effect_sizes[grna] = size
+
+        # Propagate the effect sizes
+        for grna in corr_matrix.index:
+            if grna not in grna_dict:
+                # Weighted sum of correlations with the fixed gRNAs
+                effect_sizes[grna] = np.dot(corr_matrix.loc[grna], effect_sizes) / np.sum(corr_matrix.loc[grna])
+        
+        if save:
+            # Save the effect sizes
+            effect_sizes.to_csv(os.path.join(save_folder, 'effect_sizes.csv'))
+
+        # Visualization
+        plt.figure(figsize=(10, 6))
+        sns.barplot(x=effect_sizes.index, y=effect_sizes.values, palette="viridis", hue=None, legend=False)
+
+        #for i, val in enumerate(effect_sizes.values):
+        #    plt.text(i, val + 0.02, f"{val:.2f}", ha='center', va='bottom', fontsize=9)
+        plt.title("Effect Sizes of gRNAs")
+        plt.xlabel("gRNAs")
+        plt.ylabel("Effect Size")
+        plt.xticks(rotation=45)
+        plt.tight_layout()
+        
+        if save:
+            effect_sizes_fig_path = os.path.join(save_folder, 'effect_sizes_barplot.pdf')
+            plt.savefig(effect_sizes_fig_path, dpi=300)
+        
+        plt.show()
+
+        return effect_sizes
+    
+    # Ensure the save folder exists
+    save_folder = os.path.join(os.path.dirname(csv_file), 'post_regression_analysis_results')
+    os.makedirs(save_folder, exist_ok=True)
+    
+    # Load the data
+    df = pd.read_csv(csv_file)
+    
+    # Perform analysis
+    correlation_matrix = _analyze_and_visualize_grna_correlation(df, grna_list, save_folder, save)
+    effect_sizes = _compute_effect_sizes(correlation_matrix, grna_dict, save_folder, save)