spacr 0.3.81__py3-none-any.whl → 0.4.0__py3-none-any.whl

spacr/submodules.py

@@ -1,3 +1,6 @@
+
+
+
 import seaborn as sns
 import os, random, sqlite3, re, shap
 import pandas as pd
@@ -10,7 +13,10 @@ from IPython.display import display
 from sklearn.ensemble import RandomForestClassifier
 from sklearn.inspection import permutation_importance
 from math import pi
-from scipy.stats import chi2_contingency
+from scipy.stats import chi2_contingency, pearsonr
+from scipy.spatial.distance import cosine
+
+from sklearn.metrics import mean_absolute_error
 
 import matplotlib.pyplot as plt
 from natsort import natsorted
@@ -1132,3 +1138,278 @@ def analyze_class_proportion(settings):
         print("Statistical analysis results saved.")
 
     return output
+
+def generate_score_heatmap(settings):
+    
+    def group_cv_score(csv, plate=1, column='c3', data_column='pred'):
+        
+        df = pd.read_csv(csv)
+        if 'col' in df.columns:
+            df = df[df['col']==column]
+        elif 'column' in df.columns:
+            df['col'] = df['column']
+            df = df[df['col']==column]
+        if not plate is None:
+            df['plate'] = f"plate{plate}"
+        grouped_df = df.groupby(['plate', 'row', 'col'])[data_column].mean().reset_index()
+        grouped_df['prc'] = grouped_df['plate'].astype(str) + '_' + grouped_df['row'].astype(str) + '_' + grouped_df['col'].astype(str)
+        return grouped_df
+
+    def calculate_fraction_mixed_condition(csv, plate=1, column='c3', control_sgrnas = ['TGGT1_220950_1', 'TGGT1_233460_4']):
+        df = pd.read_csv(csv)  
+        df = df[df['column_name']==column]
+        if plate not in df.columns:
+            df['plate'] = f"plate{plate}"
+        df = df[df['grna_name'].str.match(f'^{control_sgrnas[0]}$|^{control_sgrnas[1]}$')]
+        grouped_df = df.groupby(['plate', 'row_name', 'column_name'])['count'].sum().reset_index()
+        grouped_df = grouped_df.rename(columns={'count': 'total_count'})
+        merged_df = pd.merge(df, grouped_df, on=['plate', 'row_name', 'column_name'])
+        merged_df['fraction'] = merged_df['count'] / merged_df['total_count']
+        merged_df['prc'] = merged_df['plate'].astype(str) + '_' + merged_df['row_name'].astype(str) + '_' + merged_df['column_name'].astype(str)
+        return merged_df
+
+    def plot_multi_channel_heatmap(df, column='c3', cmap='coolwarm'):
+        """
+        Plot a heatmap with multiple channels as columns.
+
+        Parameters:
+        - df: DataFrame with scores for different channels.
+        - column: Column to filter by (default is 'c3').
+        """
+        # Extract row number and convert to integer for sorting
+        df['row_num'] = df['row'].str.extract(r'(\d+)').astype(int)
+
+        # Filter and sort by plate, row, and column
+        df = df[df['col'] == column]
+        df = df.sort_values(by=['plate', 'row_num', 'col'])
+
+        # Drop temporary 'row_num' column after sorting
+        df = df.drop('row_num', axis=1)
+
+        # Create a new column combining plate, row, and column for the index
+        df['plate_row_col'] = df['plate'] + '-' + df['row'] + '-' + df['col']
+
+        # Set 'plate_row_col' as the index
+        df.set_index('plate_row_col', inplace=True)
+
+        # Extract only numeric data for the heatmap
+        heatmap_data = df.select_dtypes(include=[float, int])
+
+        # Plot heatmap with square boxes, no annotations, and 'viridis' colormap
+        plt.figure(figsize=(12, 8))
+        sns.heatmap(
+            heatmap_data,
+            cmap=cmap,
+            cbar=True,
+            square=True,
+            annot=False
+        )
+
+        plt.title("Heatmap of Prediction Scores for All Channels")
+        plt.xlabel("Channels")
+        plt.ylabel("Plate-Row-Column")
+        plt.tight_layout()
+
+        # Save the figure object and return it
+        fig = plt.gcf()
+        plt.show()
+
+        return fig
+
+
+    def combine_classification_scores(folders, csv_name, data_column, plate=1, column='c3'):
+        # Ensure `folders` is a list
+        if isinstance(folders, str):
+            folders = [folders]
+
+        ls = []  # Initialize ls to store found CSV file paths
+
+        # Iterate over the provided folders
+        for folder in folders:
+            sub_folders = os.listdir(folder)  # Get sub-folder list
+            for sub_folder in sub_folders:  # Iterate through sub-folders
+                path = os.path.join(folder, sub_folder)  # Join the full path
+
+                if os.path.isdir(path):  # Check if it’s a directory
+                    csv = os.path.join(path, csv_name)  # Join path to the CSV file
+                    if os.path.exists(csv):  # If CSV exists, add to list
+                        ls.append(csv)
+                    else:
+                        print(f'No such file: {csv}')
+
+        # Initialize combined DataFrame
+        combined_df = None
+        print(f'Found {len(ls)} CSV files')
+
+        # Loop through all collected CSV files and process them
+        for csv_file in ls:
+            df = pd.read_csv(csv_file)  # Read CSV into DataFrame
+            df = df[df['col']==column]
+            if not plate is None:
+                df['plate'] = f"plate{plate}"
+            # Group the data by 'plate', 'row', and 'col'
+            grouped_df = df.groupby(['plate', 'row', 'col'])[data_column].mean().reset_index()
+            # Use the CSV filename to create a new column name
+            folder_name = os.path.dirname(csv_file).replace(".csv", "")
+            new_column_name = os.path.basename(f"{folder_name}_{data_column}")
+            print(new_column_name)
+            grouped_df = grouped_df.rename(columns={data_column: new_column_name})
+
+            # Merge into the combined DataFrame
+            if combined_df is None:
+                combined_df = grouped_df
+            else:
+                combined_df = pd.merge(combined_df, grouped_df, on=['plate', 'row', 'col'], how='outer')
+        combined_df['prc'] = combined_df['plate'].astype(str) + '_' + combined_df['row'].astype(str) + '_' + combined_df['col'].astype(str)
+        return combined_df
+    
+    def calculate_mae(df):
+        """
+        Calculate the MAE between each channel's predictions and the fraction column for all rows.
+        """
+        # Extract numeric columns excluding 'fraction' and 'prc'
+        channels = df.drop(columns=['fraction', 'prc']).select_dtypes(include=[float, int])
+
+        mae_data = []
+
+        # Compute MAE for each channel with 'fraction' for all rows
+        for column in channels.columns:
+            for index, row in df.iterrows():
+                mae = mean_absolute_error([row['fraction']], [row[column]])
+                mae_data.append({'Channel': column, 'MAE': mae, 'Row': row['prc']})
+
+        # Convert the list of dictionaries to a DataFrame
+        mae_df = pd.DataFrame(mae_data)
+        return mae_df
+
+    result_df = combine_classification_scores(settings['folders'], settings['csv_name'], settings['data_column'], settings['plate'], settings['column'], )
+    df = calculate_fraction_mixed_condition(settings['csv'], settings['plate'], settings['column'], settings['control_sgrnas'])
+    df = df[df['grna_name']==settings['fraction_grna']]
+    fraction_df = df[['fraction', 'prc']]
+    merged_df = pd.merge(fraction_df, result_df, on=['prc'])
+    cv_df = group_cv_score(settings['cv_csv'], settings['plate'], settings['column'], settings['data_column_cv'])
+    cv_df = cv_df[[settings['data_column_cv'], 'prc']]
+    merged_df = pd.merge(merged_df, cv_df, on=['prc'])
+    
+    fig = plot_multi_channel_heatmap(merged_df, settings['column'], settings['cmap'])
+    if 'row_number' in merged_df.columns:
+        merged_df = merged_df.drop('row_num', axis=1)
+    mae_df = calculate_mae(merged_df)
+    if 'row_number' in mae_df.columns:
+        mae_df = mae_df.drop('row_num', axis=1)
+        
+    if not settings['dst'] is None:
+        mae_dst = os.path.join(settings['dst'], f"mae_scores_comparison_plate_{settings['plate']}.csv")
+        merged_dst = os.path.join(settings['dst'], f"scores_comparison_plate_{settings['plate']}_data.csv")
+        heatmap_save = os.path.join(settings['dst'], f"scores_comparison_plate_{settings['plate']}.pdf")
+        mae_df.to_csv(mae_dst, index=False)
+        merged_df.to_csv(merged_dst, index=False)
+        fig.savefig(heatmap_save, format='pdf', dpi=600, bbox_inches='tight')
+    return merged_df
+
+def post_regression_analysis(csv_file, grna_dict, grna_list, save=False):
+    
+    def _analyze_and_visualize_grna_correlation(df, grna_list, save_folder, save=False):
+        """
+        Analyze and visualize the correlation matrix of gRNAs based on their fractions and overlap.
+
+        Parameters:
+        df (pd.DataFrame): DataFrame with columns ['grna', 'fraction', 'prc'].
+        grna_list (list): List of gRNAs to include in the correlation analysis.
+        save_folder (str): Path to the folder where figures and data will be saved.
+
+        Returns:
+        pd.DataFrame: Correlation matrix of the gRNAs.
+        """
+        # Filter the DataFrame to include only rows with gRNAs in the list
+        filtered_df = df[df['grna'].isin(grna_list)]
+
+        # Pivot the data to create a prc-by-gRNA matrix, using fractions as values
+        pivot_df = filtered_df.pivot_table(index='prc', columns='grna', values='fraction', aggfunc='sum').fillna(0)
+
+        # Compute the correlation matrix
+        correlation_matrix = pivot_df.corr()
+        
+        if save:
+            # Save the correlation matrix
+            correlation_matrix.to_csv(os.path.join(save_folder, 'correlation_matrix.csv'))
+        
+        # Visualize the correlation matrix as a heatmap
+        plt.figure(figsize=(10, 8))
+        sns.heatmap(correlation_matrix, annot=False, cmap='coolwarm', cbar=True)
+        plt.title('gRNA Correlation Matrix')
+        plt.xlabel('gRNAs')
+        plt.ylabel('gRNAs')
+        plt.tight_layout()
+        
+        if save:
+            correlation_fig_path = os.path.join(save_folder, 'correlation_matrix_heatmap.pdf')
+            plt.savefig(correlation_fig_path, dpi=300)
+        
+        plt.show()
+
+        return correlation_matrix
+
+    def _compute_effect_sizes(correlation_matrix, grna_dict, save_folder, save=False):
+        """
+        Compute and visualize the effect sizes of gRNAs given fixed effect sizes for a subset of gRNAs.
+
+        Parameters:
+        correlation_matrix (pd.DataFrame): Correlation matrix of gRNAs.
+        grna_dict (dict): Dictionary of gRNAs with fixed effect sizes {grna_name: effect_size}.
+        save_folder (str): Path to the folder where figures and data will be saved.
+
+        Returns:
+        pd.Series: Effect sizes of all gRNAs.
+        """
+        # Ensure the matrix is symmetric and normalize values to 0-1
+        corr_matrix = correlation_matrix.copy()
+        corr_matrix = (corr_matrix - corr_matrix.min().min()) / (corr_matrix.max().max() - corr_matrix.min().min())
+
+        # Initialize the effect sizes with dtype float
+        effect_sizes = pd.Series(0.0, index=corr_matrix.index)
+
+        # Set the effect sizes for the specified gRNAs
+        for grna, size in grna_dict.items():
+            effect_sizes[grna] = size
+
+        # Propagate the effect sizes
+        for grna in corr_matrix.index:
+            if grna not in grna_dict:
+                # Weighted sum of correlations with the fixed gRNAs
+                effect_sizes[grna] = np.dot(corr_matrix.loc[grna], effect_sizes) / np.sum(corr_matrix.loc[grna])
+        
+        if save:
+            # Save the effect sizes
+            effect_sizes.to_csv(os.path.join(save_folder, 'effect_sizes.csv'))
+
+        # Visualization
+        plt.figure(figsize=(10, 6))
+        sns.barplot(x=effect_sizes.index, y=effect_sizes.values, palette="viridis", hue=None, legend=False)
+
+        #for i, val in enumerate(effect_sizes.values):
+        #    plt.text(i, val + 0.02, f"{val:.2f}", ha='center', va='bottom', fontsize=9)
+        plt.title("Effect Sizes of gRNAs")
+        plt.xlabel("gRNAs")
+        plt.ylabel("Effect Size")
+        plt.xticks(rotation=45)
+        plt.tight_layout()
+        
+        if save:
+            effect_sizes_fig_path = os.path.join(save_folder, 'effect_sizes_barplot.pdf')
+            plt.savefig(effect_sizes_fig_path, dpi=300)
+        
+        plt.show()
+
+        return effect_sizes
+    
+    # Ensure the save folder exists
+    save_folder = os.path.join(os.path.dirname(csv_file), 'post_regression_analysis_results')
+    os.makedirs(save_folder, exist_ok=True)
+    
+    # Load the data
+    df = pd.read_csv(csv_file)
+    
+    # Perform analysis
+    correlation_matrix = _analyze_and_visualize_grna_correlation(df, grna_list, save_folder, save)
+    effect_sizes = _compute_effect_sizes(correlation_matrix, grna_dict, save_folder, save)

spacr/toxo.py

@@ -24,117 +24,128 @@ from sklearn.metrics import mean_absolute_error
 
 from matplotlib.gridspec import GridSpec
 
+def custom_volcano_plot(data_path, metadata_path, metadata_column='tagm_location',point_size=50, figsize=20, threshold=0,save_path=None, x_lim=[-0.5, 0.5], y_lims=[[0, 6], [9, 20]]):
 
-def custom_volcano_plot(data_path, metadata_path, metadata_column='tagm_location',
-                        point_size=50, figsize=20, threshold=0,
-                        save_path=None, x_lim=[-0.5, 0.5], y_lims=[[0, 6], [9, 15]]):
+    # Dictionary mapping compartment to color
     
-    markers = [
-        'o',  # Circle
-        'X',  # X-shaped marker
-        '^',  # Upward triangle
-        's',  # Square
-        'v',  # Downward triangle
-        'P',  # Plus-filled pentagon
-        '*',  # Star
-        '+',  # Plus
-        'x',  # Cross
-        '.',  # Point
-        ',',  # Pixel
-        'd',  # Diamond
-        'D',  # Thin diamond
-        'h',  # Hexagon 1
-        'H',  # Hexagon 2
-        'p',  # Pentagon
-        '|',  # Vertical line
-        '_',  # Horizontal line
-    ]
+    colors = {'micronemes':'black',
+              'rhoptries 1':'darkviolet',
+              'rhoptries 2':'darkviolet',
+              'nucleus - chromatin':'blue',
+              'nucleus - non-chromatin':'blue',
+              'dense granules':'teal',
+              'ER 1':'pink',
+              'ER 2':'pink',
+              'unknown':'black',
+              'tubulin cytoskeleton':'slategray',
+              'IMC':'slategray',
+              'PM - peripheral 1':'slategray',
+              'PM - peripheral 2':'slategray',
+              'cytosol':'turquoise',
+              'mitochondrion - soluble':'red',
+              'mitochondrion - membranes':'red',
+              'apicoplast':'slategray',
+              'Golgi':'green',
+              'PM - integral':'slategray',
+              'apical 1':'orange',
+              'apical 2':'orange',
+              '19S proteasome':'slategray',
+              '20S proteasome':'slategray',
+              '60S ribosome':'slategray',
+              '40S ribosome':'slategray',
+              }
+
+    # Increase font size for better readability
+    fontsize = 18
+    plt.rcParams.update({'font.size': fontsize})
     
-    plt.rcParams.update({'font.size': 14})
-
-    # Load data
+    # --- Load data ---
     if isinstance(data_path, pd.DataFrame):
         data = data_path
     else:
         data = pd.read_csv(data_path)
-
-    fontsize = 18
-
-    plt.rcParams.update({'font.size': fontsize})
+        
+    # Extract ‘variable’ and ‘gene_nr’ from your feature notation
     data['variable'] = data['feature'].str.extract(r'\[(.*?)\]')
     data['variable'].fillna(data['feature'], inplace=True)
     data['gene_nr'] = data['variable'].str.split('_').str[0]
     data = data[data['variable'] != 'Intercept']
 
-    # Load metadata
+    # --- Load metadata ---
     if isinstance(metadata_path, pd.DataFrame):
         metadata = metadata_path
     else:
         metadata = pd.read_csv(metadata_path)
+
     metadata['gene_nr'] = metadata['gene_nr'].astype(str)
     data['gene_nr'] = data['gene_nr'].astype(str)
 
-    merged_data = pd.merge(data, metadata[['gene_nr', metadata_column]], on='gene_nr', how='left')
+    # Merge data and metadata
+    merged_data = pd.merge(data, metadata[['gene_nr', metadata_column]], 
+                           on='gene_nr', how='left')
     merged_data[metadata_column].fillna('unknown', inplace=True)
 
-    # Define palette and markers
-    palette = {'pc': 'red', 'nc': 'green', 'control': 'white', 'other': 'gray'}
-    marker_dict = {val: marker for val, marker in zip(
-        merged_data[metadata_column].unique(), markers)}
-
-    # Create the figure with custom spacing
-    fig = plt.figure(figsize=(figsize,figsize))
+    # --- Create figure with "upper" and "lower" subplots sharing the x-axis ---
+    fig = plt.figure(figsize=(figsize, figsize))
     gs = GridSpec(2, 1, height_ratios=[1, 3], hspace=0.05)
-
     ax_upper = fig.add_subplot(gs[0])
     ax_lower = fig.add_subplot(gs[1], sharex=ax_upper)
 
     # Hide x-axis labels on the upper plot
     ax_upper.tick_params(axis='x', which='both', bottom=False, labelbottom=False)
 
+    # List to collect the variables (hits) that meet threshold criteria
     hit_list = []
 
-    # Scatter plot on both axes
+    # --- Scatter plot on both axes ---
     for _, row in merged_data.iterrows():
         y_val = -np.log10(row['p_value'])
+        
+        # Decide which axis to draw on based on the p-value
         ax = ax_upper if y_val > y_lims[1][0] else ax_lower
 
+        # Here is the main change: color by the colors dict
         ax.scatter(
-            row['coefficient'], y_val,
-            color=palette.get(row['condition'], 'gray'),
-            marker=marker_dict.get(row[metadata_column], 'o'),
-            s=point_size, edgecolor='black', alpha=0.6
+            row['coefficient'], 
+            y_val,
+            color=colors.get(row[metadata_column], 'gray'),  # <-- Use your color dict
+            marker='o',  # You can fix a single marker if desired
+            s=point_size, 
+            edgecolor='black', 
+            alpha=0.6
         )
 
-        if row['p_value'] <= 0.05 and abs(row['coefficient']) >= abs(threshold):
+        # Check significance thresholds
+        if (row['p_value'] <= 0.05) and (abs(row['coefficient']) >= abs(threshold)):
             hit_list.append(row['variable'])
 
-    # Set axis limits
+    # --- Adjust axis limits ---
     ax_upper.set_ylim(y_lims[1])
     ax_lower.set_ylim(y_lims[0])
     ax_lower.set_xlim(x_lim)
 
+    # Hide top spines
     ax_lower.spines['top'].set_visible(False)
     ax_upper.spines['top'].set_visible(False)
     ax_upper.spines['bottom'].set_visible(False)
 
-    # Set x-axis and y-axis titles
-    ax_lower.set_xlabel('Coefficient')  # X-axis title on the lower graph
-    ax_lower.set_ylabel('-log10(p-value)')  # Y-axis title on the lower graph
-    ax_upper.set_ylabel('-log10(p-value)')  # Y-axis title on the upper graph
-    
+    # Set x-axis and y-axis labels
+    ax_lower.set_xlabel('Coefficient')  
+    ax_lower.set_ylabel('-log10(p-value)')
+    ax_upper.set_ylabel('-log10(p-value)')
+
     for ax in [ax_upper, ax_lower]:
         ax.spines['right'].set_visible(False)
 
-    # Add threshold lines to both axes
+    # --- Add threshold lines to both axes ---
     for ax in [ax_upper, ax_lower]:
         ax.axvline(x=-abs(threshold), linestyle='--', color='black')
         ax.axvline(x=abs(threshold), linestyle='--', color='black')
 
     ax_lower.axhline(y=-np.log10(0.05), linestyle='--', color='black')
 
-    # Annotate significant points
-    texts_upper, texts_lower = [], []  # Collect text annotations separately
+    # --- Annotate significant points ---
+    texts_upper, texts_lower = [], []
 
     for _, row in merged_data.iterrows():
         y_val = -np.log10(row['p_value'])
@@ -142,38 +153,50 @@ def custom_volcano_plot(data_path, metadata_path, metadata_column='tagm_location
             continue
 
         ax = ax_upper if y_val > y_lims[1][0] else ax_lower
-        text = ax.text(row['coefficient'], y_val, row['variable'], 
-                       fontsize=fontsize, ha='center', va='bottom')
+        text = ax.text(
+            row['coefficient'],
+            y_val,
+            row['variable'],
+            fontsize=fontsize,
+            ha='center',
+            va='bottom'
+        )
 
         if ax == ax_upper:
             texts_upper.append(text)
         else:
             texts_lower.append(text)
 
-    # Adjust text positions to avoid overlap
+    # Attempt to keep text labels from overlapping
     adjust_text(texts_upper, ax=ax_upper, arrowprops=dict(arrowstyle='-', color='black'))
     adjust_text(texts_lower, ax=ax_lower, arrowprops=dict(arrowstyle='-', color='black'))
 
-    # Add a single legend on the lower axis
-    handles = [plt.Line2D([0], [0], marker=m, color='w', markerfacecolor='gray', markersize=10)
-               for m in marker_dict.values()]
-    labels = marker_dict.keys()
-    ax_lower.legend(handles, 
-                    labels, 
-                    bbox_to_anchor=(1.05, 1), 
-                    loc='upper left',
-                    borderaxespad=0.25, 
-                    labelspacing=2, 
-                    handletextpad=0.25, 
-                    markerscale=2, 
-                    prop={'size': fontsize})
-
-
-    # Save and show the plot
+    # --- Add a legend keyed by color (optional) ---
+    # If you'd like a legend that shows what each compartment color represents:
+    legend_handles = []
+    for comp, comp_color in colors.items():
+        # Create a “dummy” scatter for legend
+        legend_handles.append(
+            plt.Line2D([0], [0], marker='o', color=comp_color, 
+                       label=comp, linewidth=0, markersize=8)
+        )
+    # You can adjust the location and styling of the legend to taste:
+    ax_lower.legend(
+        handles=legend_handles,
+        bbox_to_anchor=(1.05, 1),
+        loc='upper left',
+        borderaxespad=0.25,
+        labelspacing=2,
+        handletextpad=0.25,
+        markerscale=1.5,
+        prop={'size': fontsize}
+    )
+
+    # --- Save and show ---
     if save_path:
         plt.savefig(save_path, format='pdf', bbox_inches='tight')
     plt.show()
-    
+
     return hit_list
 
 def go_term_enrichment_by_column(significant_df, metadata_path, go_term_columns=['Computed GO Processes', 'Curated GO Components', 'Curated GO Functions', 'Curated GO Processes']):