spacr 0.0.82__py3-none-any.whl → 0.1.1__py3-none-any.whl

spacr/plot.py

@@ -1110,7 +1110,7 @@ def _imshow(img, labels, nrow=20, color='white', fontsize=12):
             idx = i * n_col + j
             if idx < n_images:
                 canvas[i * img_height:(i + 1) * img_height, j * img_width:(j + 1) * img_width] = np.transpose(img[idx], (1, 2, 0))        
-    plt.figure(figsize=(50, 50))
+    fig = plt.figure(figsize=(50, 50))
     plt.imshow(canvas)
     plt.axis("off")
     for i, label in enumerate(labels):
@@ -1119,7 +1119,7 @@ def _imshow(img, labels, nrow=20, color='white', fontsize=12):
         x = col * img_width + 2
         y = row * img_height + 15
         plt.text(x, y, label, color=color, fontsize=fontsize, fontweight='bold')
-    plt.show()
+    return fig
     
 def _plot_histograms_and_stats(df):
     conditions = df['condition'].unique()
@@ -1218,8 +1218,7 @@ def generate_plate_heatmap(df, plate_number, variable, grouping, min_max, min_co
         df = df[df['count'] >= min_count]
 
     # Explicitly set observed=True to avoid FutureWarning
-    grouped = df.groupby(['row', 'col'], observed=True) 
-
+    grouped = df.groupby(['row', 'col'], observed=True) # Group by row and column
     
     if grouping == 'mean':
         plate = grouped[variable].mean().reset_index()
@@ -1245,7 +1244,7 @@ def generate_plate_heatmap(df, plate_number, variable, grouping, min_max, min_co
         
     return plate_map, min_max
 
-def _plot_plates(df, variable, grouping, min_max, cmap, min_count=0):
+def plot_plates(df, variable, grouping, min_max, cmap, min_count=0, verbose=True):
     plates = df['prc'].str.split('_', expand=True)[0].unique()
     n_rows, n_cols = (len(plates) + 3) // 4, 4
     fig, ax = plt.subplots(n_rows, n_cols, figsize=(40, 5 * n_rows))
@@ -1260,7 +1259,8 @@ def _plot_plates(df, variable, grouping, min_max, cmap, min_count=0):
         fig.delaxes(ax[i])
     
     plt.subplots_adjust(wspace=0.1, hspace=0.4)
-    plt.show()
+    if verbose:
+        plt.show()
     return fig
 
 def print_mask_and_flows(stack, mask, flows, overlay=False):
@@ -1470,3 +1470,99 @@ def plot_object_outlines(src, objects=['nucleus','cell','pathogen'], channels=[0
                                overlay=True,
                                max_nr=10,
                                randomize=True)
+        
+def volcano_plot(coef_df, filename='volcano_plot.pdf'):
+    # Create the volcano plot
+    plt.figure(figsize=(10, 6))
+    sns.scatterplot(
+        data=coef_df, 
+        x='coefficient', 
+        y='-log10(p_value)', 
+        hue='highlight', 
+        palette={True: 'red', False: 'blue'}
+    )
+    plt.title('Volcano Plot of Coefficients')
+    plt.xlabel('Coefficient')
+    plt.ylabel('-log10(p-value)')
+    plt.axhline(y=-np.log10(0.05), color='red', linestyle='--')
+    plt.legend().remove()
+    plt.savefig(filename, format='pdf')
+    print(f'Saved Volcano plot: {filename}')
+    plt.show()
+
+def plot_histogram(df, dependent_variable):
+    # Plot histogram of the dependent variable
+    plt.figure(figsize=(10, 6))
+    sns.histplot(df[dependent_variable], kde=True)
+    plt.title(f'Histogram of {dependent_variable}')
+    plt.xlabel(dependent_variable)
+    plt.ylabel('Frequency')
+    plt.show()
+
+def plot_lorenz_curves(csv_files, remove_keys=['TGGT1_220950_1', 'TGGT1_233460_4']):
+    
+    def lorenz_curve(data):
+        """Calculate Lorenz curve."""
+        sorted_data = np.sort(data)
+        cumulative_data = np.cumsum(sorted_data)
+        lorenz_curve = cumulative_data / cumulative_data[-1]
+        lorenz_curve = np.insert(lorenz_curve, 0, 0)
+        return lorenz_curve
+    
+    combined_data = []
+
+    plt.figure(figsize=(10, 6))
+
+    for idx, csv_file in enumerate(csv_files):
+        if idx == 1:
+            save_fldr = os.path.dirname(csv_file)
+            save_path = os.path.join(save_fldr, 'lorenz_curve.pdf')
+            
+        df = pd.read_csv(csv_file)
+        for remove in remove_keys:
+            df = df[df['key'] != remove]
+        
+        values = df['value'].values
+        combined_data.extend(values)
+        
+        lorenz = lorenz_curve(values)
+        name = os.path.basename(csv_file)[:3]
+        plt.plot(np.linspace(0, 1, len(lorenz)), lorenz, label=name)
+
+    # Plot combined Lorenz curve
+    combined_lorenz = lorenz_curve(np.array(combined_data))
+    plt.plot(np.linspace(0, 1, len(combined_lorenz)), combined_lorenz, label="Combined Lorenz Curve", linestyle='--', color='black')
+    
+    plt.title('Lorenz Curves')
+    plt.xlabel('Cumulative Share of Individuals')
+    plt.ylabel('Cumulative Share of Value')
+    plt.legend()
+    plt.grid(False)
+    plt.savefig(save_path)
+    plt.show()
+
+def plot_permutation(permutation_df):
+    num_features = len(permutation_df)
+    fig_height = max(8, num_features * 0.3)  # Set a minimum height of 8 and adjust height based on number of features
+    fig_width = 10  # Width can be fixed or adjusted similarly
+    font_size = max(10, 12 - num_features * 0.2)  # Adjust font size dynamically
+
+    fig, ax = plt.subplots(figsize=(fig_width, fig_height))
+    ax.barh(permutation_df['feature'], permutation_df['importance_mean'], xerr=permutation_df['importance_std'], color="teal", align="center", alpha=0.6)
+    ax.set_xlabel('Permutation Importance', fontsize=font_size)
+    ax.tick_params(axis='both', which='major', labelsize=font_size)
+    plt.tight_layout()
+    return fig
+
+def plot_feature_importance(feature_importance_df):
+    num_features = len(feature_importance_df)
+    fig_height = max(8, num_features * 0.3)  # Set a minimum height of 8 and adjust height based on number of features
+    fig_width = 10  # Width can be fixed or adjusted similarly
+    font_size = max(10, 12 - num_features * 0.2)  # Adjust font size dynamically
+
+    fig, ax = plt.subplots(figsize=(fig_width, fig_height))
+    ax.barh(feature_importance_df['feature'], feature_importance_df['importance'], color="blue", align="center", alpha=0.6)
+    ax.set_xlabel('Feature Importance', fontsize=font_size)
+    ax.tick_params(axis='both', which='major', labelsize=font_size)
+    plt.tight_layout()
+    return fig

spacr/sequencing.py

@@ -10,6 +10,7 @@ import statsmodels.api as sm
 from statsmodels.regression.mixed_linear_model import MixedLM
 from statsmodels.stats.outliers_influence import variance_inflation_factor
 from scipy.stats import gmean
+from scipy import stats
 from difflib import SequenceMatcher
 from collections import Counter
 from IPython.display import display
@@ -305,13 +306,10 @@ def analyze_reads(settings):
                 qc_df = pd.DataFrame([qc])
                 qc_df.to_csv(qc_file_path, index=False)
                 
-    settings.setdefault('upstream', 'CTTCTGGTAAATGGGGATGTCAAGTT') 
-    settings.setdefault('downstream', 'GTTTAAGAGCTATGCTGGAAACAGCAG') #This is the reverce compliment of the column primer starting from the end #TGCTGTTTAAGAGCTATGCTGGAAACAGCA
-    settings.setdefault('barecode_length_1', 8)
-    settings.setdefault('barecode_length_2', 7)
-    settings.setdefault('chunk_size', 1000000)
-    settings.setdefault('test', False)
-    
+    from .utils import get_analyze_reads_default_settings
+
+    settings = get_analyze_reads_default_settings(settings)
+
     samples_dict = parse_gz_files(settings['src'])
     combine_reads(samples_dict, settings['src'], settings['chunk_size'], settings['barecode_length_1'], settings['barecode_length_2'], settings['upstream'], settings['downstream'])
 
@@ -477,17 +475,10 @@ def map_barcodes(h5_file_path, settings={}):
             #print(f"Max count for {nc} in other columns: {max_count_c3}")
             
         return filtered_df
+    
+    from .settings import get_map_barcodes_default_settings
 
-    settings.setdefault('grna', '/home/carruthers/Documents/grna_barcodes.csv')
-    settings.setdefault('barcodes', '/home/carruthers/Documents/SCREEN_BARCODES.csv')
-    settings.setdefault('plate_dict', {'EO1': 'plate1', 'EO2': 'plate2', 'EO3': 'plate3', 'EO4': 'plate4', 'EO5': 'plate5', 'EO6': 'plate6', 'EO7': 'plate7', 'EO8': 'plate8'})
-    settings.setdefault('test', False)
-    settings.setdefault('verbose', True)
-
-    settings.setdefault('pc', 'TGGT1_220950_1')
-    settings.setdefault('pc_loc', 'c2')
-    settings.setdefault('nc', 'TGGT1_233460_4')
-    settings.setdefault('nc_loc', 'c1')
+    settings = get_map_barcodes_default_settings(settings)
 
     fldr = os.path.splitext(h5_file_path)[0]
     file_name = os.path.basename(fldr)
@@ -1393,16 +1384,6 @@ def generate_fraction_map(df, gene_column, min_=10, plates=['p1','p2','p3','p4']
     independent_variables.index.name = 'prc'
     independent_variables = independent_variables.loc[:, (independent_variables.sum() != 0)]
     return independent_variables
-
-
-def plot_histogram(df, dependent_variable):
-    # Plot histogram of the dependent variable
-    plt.figure(figsize=(10, 6))
-    sns.histplot(df[dependent_variable], kde=True)
-    plt.title(f'Histogram of {dependent_variable}')
-    plt.xlabel(dependent_variable)
-    plt.ylabel('Frequency')
-    plt.show()
     
 def precess_reads(csv_path, fraction_threshold, plate):
     # Read the CSV file into a DataFrame
@@ -1475,32 +1456,42 @@ def check_normality(data, variable_name, verbose=False):
             print(f"The data for {variable_name} is not normally distributed.")
         return False
 
-def process_scores(df, dependent_variable, plate, min_cell_count=25, agg_type='mean', transform=None):
+def process_scores(df, dependent_variable, plate, min_cell_count=25, agg_type='mean', transform=None, regression_type='ols'):
     
     if plate is not None:
         df['plate'] = plate
 
+    if 'col' not in df.columns:
+        df['col'] = df['column']
+
     df['prc'] = df['plate'] + '_' + df['row'] + '_' + df['col']
     df = df[['prc', dependent_variable]]
 
     # Group by prc and calculate the mean and count of the dependent_variable
     grouped = df.groupby('prc')[dependent_variable]
-
-    print(f'Using agg_type: {agg_type}')
-    if agg_type == 'median':
-        dependent_df = grouped.median().reset_index()
-    elif agg_type == 'mean':
-        dependent_df = grouped.mean().reset_index()
-    elif agg_type == 'quantile':
-        dependent_df = grouped.quantile(0.75).reset_index()
-    elif agg_type == None:
-        dependent_df = df.reset_index()
-        if 'prcfo' in dependent_df.columns:
-            dependent_df = dependent_df.drop(columns=['prcfo'])
-
-    else:
-        raise ValueError(f"Unsupported aggregation type {agg_type}")
-
+    
+    if regression_type != 'poisson':
+    
+        print(f'Using agg_type: {agg_type}')
+
+        if agg_type == 'median':
+            dependent_df = grouped.median().reset_index()
+        elif agg_type == 'mean':
+            dependent_df = grouped.mean().reset_index()
+        elif agg_type == 'quantile':
+            dependent_df = grouped.quantile(0.75).reset_index()
+        elif agg_type == None:
+            dependent_df = df.reset_index()
+            if 'prcfo' in dependent_df.columns:
+                dependent_df = dependent_df.drop(columns=['prcfo'])
+        else:
+            raise ValueError(f"Unsupported aggregation type {agg_type}")
+            
+    if regression_type == 'poisson':
+        agg_type = 'count'
+        print(f'Using agg_type: {agg_type} for poisson regression')
+        dependent_df = grouped.sum().reset_index()        
+        
     # Calculate cell_count for all cases
     cell_count = grouped.size().reset_index(name='cell_count')
 
@@ -1516,7 +1507,7 @@ def process_scores(df, dependent_variable, plate, min_cell_count=25, agg_type='m
     if not transform is None:
         transformer = apply_transformation(dependent_df[dependent_variable], transform=transform)
         transformed_var = f'{transform}_{dependent_variable}'
-        df[transformed_var] = transformer.fit_transform(dependent_df[[dependent_variable]])
+        dependent_df[transformed_var] = transformer.fit_transform(dependent_df[[dependent_variable]])
         dependent_variable = transformed_var
         is_normal = check_normality(dependent_df[transformed_var], transformed_var)
 
@@ -1557,6 +1548,7 @@ def regression_model(X, y, regression_type='ols', groups=None, alpha=1.0, remove
         model = sm.GLS(y, X).fit()
 
     elif regression_type == 'wls':
+        weights = 1 / np.sqrt(X.iloc[:, 1])
         model = sm.WLS(y, X, weights=weights).fit()
 
     elif regression_type == 'rlm':
@@ -1589,7 +1581,7 @@ def regression_model(X, y, regression_type='ols', groups=None, alpha=1.0, remove
         model = sm.Probit(y, X).fit()
 
     elif regression_type == 'poisson':
-        model_poisson = sm.Poisson(y, X).fit()
+        model = sm.Poisson(y, X).fit()
 
     elif regression_type == 'lasso':
         model = Lasso(alpha=alpha).fit(X, y)
@@ -1611,25 +1603,6 @@ def regression_model(X, y, regression_type='ols', groups=None, alpha=1.0, remove
 
     return model
     
-def volcano_plot(coef_df, filename='volcano_plot.pdf'):
-    # Create the volcano plot
-    plt.figure(figsize=(10, 6))
-    sns.scatterplot(
-        data=coef_df, 
-        x='coefficient', 
-        y='-log10(p_value)', 
-        hue='highlight', 
-        palette={True: 'red', False: 'blue'}
-    )
-    plt.title('Volcano Plot of Coefficients')
-    plt.xlabel('Coefficient')
-    plt.ylabel('-log10(p-value)')
-    plt.axhline(y=-np.log10(0.05), color='red', linestyle='--')
-    plt.legend().remove()
-    plt.savefig(filename, format='pdf')
-    print(f'Saved Volcano plot: {filename}')
-    plt.show()
-    
 def clean_controls(df,pc,nc,other):
     if 'col' in df.columns:
         df['column'] = df['col']
@@ -1642,20 +1615,60 @@ def clean_controls(df,pc,nc,other):
         print(f'Removed data from {nc, pc, other}')
     return df
 
+# Remove outliers by capping values at 1st and 99th percentiles for numerical columns only
+def remove_outliers(df, low=0.01, high=0.99):
+    numerical_cols = df.select_dtypes(include=[np.number]).columns
+    quantiles = df[numerical_cols].quantile([low, high])
+    for col in numerical_cols:
+        df[col] = np.clip(df[col], quantiles.loc[low, col], quantiles.loc[high, col])
+    return df
+
+def calculate_p_values(X, y, model):
+    # Predict y values
+    y_pred = model.predict(X)
+    
+    # Calculate residuals
+    residuals = y - y_pred
+    
+    # Calculate the standard error of the residuals
+    dof = X.shape[0] - X.shape[1] - 1
+    residual_std_error = np.sqrt(np.sum(residuals ** 2) / dof)
+    
+    # Calculate the standard error of the coefficients
+    X_design = np.hstack((np.ones((X.shape[0], 1)), X))  # Add intercept
+    
+    # Use pseudoinverse instead of inverse to handle singular matrices
+    coef_var_covar = residual_std_error ** 2 * np.linalg.pinv(X_design.T @ X_design)
+    coef_standard_errors = np.sqrt(np.diag(coef_var_covar))
+    
+    # Calculate t-statistics
+    t_stats = model.coef_ / coef_standard_errors[1:]  # Skip intercept error
+    
+    # Calculate p-values
+    p_values = [2 * (1 - stats.t.cdf(np.abs(t), dof)) for t in t_stats]
+    
+    return np.array(p_values)  # Ensure p_values is a 1-dimensional array
+
 def regression(df, csv_path, dependent_variable='predictions', regression_type=None, alpha=1.0, remove_row_column_effect=False):
 
+    from .plot import volcano_plot, plot_histogram
+
     volcano_filename = os.path.splitext(os.path.basename(csv_path))[0] + '_volcano_plot.pdf'
     volcano_filename = regression_type+'_'+volcano_filename
     if regression_type == 'quantile':
         volcano_filename = str(alpha)+'_'+volcano_filename
     volcano_path=os.path.join(os.path.dirname(csv_path), volcano_filename)
 
+    is_normal = check_normality(df[dependent_variable], dependent_variable)
+
     if regression_type is None:
         if is_normal:
             regression_type = 'ols'
         else:
             regression_type = 'glm'
 
+    #df = remove_outliers(df)
+
     if remove_row_column_effect:
 
         ## 1. Fit the initial model with row and column to estimate their effects
@@ -1714,7 +1727,7 @@ def regression(df, csv_path, dependent_variable='predictions', regression_type=N
     model = regression_model(X, y, regression_type=regression_type, groups=groups, alpha=alpha, remove_row_column_effect=remove_row_column_effect)
     
     # Get the model coefficients and p-values
-    if regression_type in ['ols','gls','wls','rlm','glm','mixed','quantile','logit','probit','poisson','lasso','ridge']:
+    if regression_type in ['ols','gls','wls','rlm','glm','mixed','quantile','logit','probit','poisson']:
         coefs = model.params
         p_values = model.pvalues
 
@@ -1723,6 +1736,18 @@ def regression(df, csv_path, dependent_variable='predictions', regression_type=N
             'coefficient': coefs.values,
             'p_value': p_values.values
         })
+    elif regression_type in ['ridge', 'lasso']:
+        coefs = model.coef_
+        coefs = np.array(coefs).flatten()
+        # Calculate p-values
+        p_values = calculate_p_values(X, y, model)
+        p_values = np.array(p_values).flatten()
+
+        # Create a DataFrame for the coefficients and p-values
+        coef_df = pd.DataFrame({
+            'feature': X.columns,
+            'coefficient': coefs,
+            'p_value': p_values})
     else:
         coefs = model.coef_
         intercept = model.intercept_
@@ -1745,31 +1770,31 @@ def regression(df, csv_path, dependent_variable='predictions', regression_type=N
 
     return model, coef_df
 
-def set_regression_defaults(settings):
-    settings.setdefault('gene_weights_csv', '/nas_mnt/carruthers/Einar/mitoscreen/sequencing/combined_reads/EO1_combined/EO1_combined_combination_counts.csv')
-    settings.setdefault('dependent_variable','predictions')
-    settings.setdefault('transform',None)
-    settings.setdefault('agg_type','mean')
-    settings.setdefault('min_cell_count',25)
-    settings.setdefault('regression_type','ols')
-    settings.setdefault('remove_row_column_effect',False)
-    settings.setdefault('alpha',1)
-    settings.setdefault('fraction_threshold',0.1)
-    settings.setdefault('nc','c1')
-    settings.setdefault('pc','c2')
-    settings.setdefault('other','c3')
-    settings.setdefault('plate','plate1')
-    
-    if settings['regression_type'] == 'quantile':
-        print(f"Using alpha as quantile for quantile regression, alpha: {settings['alpha']}")
-        settings['agg_type'] = None
-        print(f'agg_type set to None for quantile regression')
-    return settings
-
 def perform_regression(df, settings):
+
+    from spacr.plot import plot_plates
+    from .utils import merge_regression_res_with_metadata
+    from .settings import get_perform_regression_default_settings
+
+    reg_types = ['ols','gls','wls','rlm','glm','mixed','quantile','logit','probit','poisson','lasso','ridge']
+    if settings['regression_type'] not in reg_types:
+        print(f'Possible regression types: {reg_types}')
+        raise ValueError(f"Unsupported regression type {settings['regression_type']}")
+
+    if isinstance(df, str):
+        df = pd.read_csv(df)
+    elif isinstance(df, pd.DataFrame):
+        pass
+    else:
+        raise ValueError("Data must be a DataFrame or a path to a CSV file")
     
-    from spacr.plot import _plot_plates
     
+    if settings['dependent_variable'] not in df.columns:
+        print(f'Columns in DataFrame:')
+        for col in df.columns:
+            print(col)
+        raise ValueError(f"Dependent variable {settings['dependent_variable']} not found in the DataFrame")
+        
     results_filename = os.path.splitext(os.path.basename(settings['gene_weights_csv']))[0] + '_results.csv'
     hits_filename = os.path.splitext(os.path.basename(settings['gene_weights_csv']))[0] + '_results_significant.csv'
     
@@ -1781,10 +1806,22 @@ def perform_regression(df, settings):
     results_path=os.path.join(os.path.dirname(settings['gene_weights_csv']), results_filename)
     hits_path=os.path.join(os.path.dirname(settings['gene_weights_csv']), hits_filename)
     
-    settings = set_regression_defaults(settings)
+    settings = get_perform_regression_default_settings(settings)
+
+    settings_df = pd.DataFrame(list(settings.items()), columns=['Key', 'Value'])
+    settings_dir = os.path.dirname(settings['gene_weights_csv'])
+    settings_csv = os.path.join(settings_dir,f"{settings['regression_type']}_regression_settings.csv")
+    settings_df.to_csv(settings_csv, index=False)
+    display(settings_df)
     
     df = clean_controls(df,settings['pc'],settings['nc'],settings['other'])
+
+    if 'prediction_probability_class_1' in df.columns:
+        if not settings['class_1_threshold'] is None:
+            df['predictions'] = (df['prediction_probability_class_1'] >= settings['class_1_threshold']).astype(int)
+
     dependent_df, dependent_variable = process_scores(df, settings['dependent_variable'], settings['plate'], settings['min_cell_count'], settings['agg_type'], settings['transform'])
+    
     display(dependent_df)
     
     independent_df = precess_reads(settings['gene_weights_csv'], settings['fraction_threshold'], settings['plate'])
@@ -1793,8 +1830,9 @@ def perform_regression(df, settings):
     merged_df = pd.merge(independent_df, dependent_df, on='prc')
     
     merged_df[['plate', 'row', 'column']] = merged_df['prc'].str.split('_', expand=True)
-        
-    plate_heatmap = _plot_plates(df, variable=dependent_variable, grouping='mean', min_max='allq', cmap='viridis', min_count=settings['min_cell_count'])                
+    
+    if settings['transform'] is None:
+        _ = plot_plates(df, variable=dependent_variable, grouping='mean', min_max='allq', cmap='viridis', min_count=settings['min_cell_count'])                
 
     model, coef_df = regression(merged_df, settings['gene_weights_csv'], dependent_variable, settings['regression_type'], settings['alpha'], settings['remove_row_column_effect'])
     
@@ -1813,6 +1851,17 @@ def perform_regression(df, settings):
         print(model.summary())
     
     significant.to_csv(hits_path, index=False)
+
+    me49 = '/home/carruthers/Documents/TGME49_Summary.csv'
+    gt1 = '/home/carruthers/Documents/TGGT1_Summary.csv'
+
+    _ = merge_regression_res_with_metadata(hits_path, me49, name='_me49_metadata')
+    _ = merge_regression_res_with_metadata(hits_path, gt1, name='_gt1_metadata')
+    _ = merge_regression_res_with_metadata(results_path, me49, name='_me49_metadata')
+    _ = merge_regression_res_with_metadata(results_path, gt1, name='_gt1_metadata')
+
     print('Significant Genes')
     display(significant)
     return coef_df
+
+