spacr 0.3.38__py3-none-any.whl → 0.3.42__py3-none-any.whl

spacr/core.py

@@ -143,7 +143,7 @@ def preprocess_generate_masks(src, settings={}):
                                 start = time.time()
                                 if i+1 <= settings['examples_to_plot']:
                                     file_path = os.path.join(merged_src, file)
-                                    plot_image_mask_overlay(file_path, settings['channels'], settings['cell_channel'], settings['nucleus_channel'], settings['pathogen_channel'], figuresize=10, normalize=True, thickness=3, save_pdf=True)
+                                    plot_image_mask_overlay(file_path, settings['channels'], settings['cell_channel'], settings['nucleus_channel'], settings['pathogen_channel'], figuresize=10, percentiles=(1,99), thickness=3, save_pdf=True)
                                     stop = time.time()
                                     duration = stop-start
                                     time_ls.append(duration)

spacr/io.py

@@ -1686,11 +1686,16 @@ def preprocess_img_data(settings):
         print(f'Found {extension_counts[most_common_extension]} {most_common_extension} files')
     else:
         print(f'Could not find any {valid_ext} files in {src} only found {extension_counts[0]}')
-        if os.path.exists(src+'/stack'):
+        
+        
+        
+        
+        
+        if os.path.exists(os.path.join(src,'stack')):
             print('Found existing stack folder.')
-        if os.path.exists(src+'/channel_stack'):
+        if os.path.exists(os.path.join(src,'channel_stack')):
             print('Found existing channel_stack folder.')
-        if os.path.exists(src+'/norm_channel_stack'):
+        if os.path.exists(os.path.join(src,'norm_channel_stack')):
             print('Found existing norm_channel_stack folder. Skipping preprocessing')
             return settings, src
         
@@ -1713,12 +1718,13 @@ def preprocess_img_data(settings):
 
         src = _run_test_mode(settings['src'], regex, timelapse, test_images, random_test)
         settings['src'] = src
-
+    
+    stack_path = os.path.join(src, 'stack')
     if img_format == None:
-        if not os.path.exists(src+'/stack'):
+        if not os.path.exists(stack_path):
             _merge_channels(src, plot=False)   
-    
-    if not os.path.exists(src+'/stack'):
+   
+    if not os.path.exists(stack_path):
         try:
             if not img_format == None:
                 if timelapse:
@@ -1727,7 +1733,7 @@ def preprocess_img_data(settings):
                     _rename_and_organize_image_files(src, regex, batch_size, pick_slice, skip_mode, metadata_type, img_format)
                     
                     #Make sure no batches will be of only one image
-                    all_imgs = len(src+'/stack')
+                    all_imgs = len(stack_path)
                     full_batches = all_imgs // batch_size
                     last_batch_size = all_imgs % batch_size
                     
@@ -1738,26 +1744,27 @@ def preprocess_img_data(settings):
                             raise ValueError("Only one batch of size 1 detected. Adjust the batch size.")
                         # If the last batch is of size 1, merge it with the second last batch
                         elif full_batches > 0:
+                            print(f"all images: {all_imgs},  full batch: {full_batches}, last batch: {last_batch_size}")
                             raise ValueError("Last batch of size 1 detected. Adjust the batch size.")
         
                 _merge_channels(src, plot=False)
 
                 if timelapse:
-                    _create_movies_from_npy_per_channel(src+'/stack', fps=2)
+                    _create_movies_from_npy_per_channel(stack_path, fps=2)
 
                 if plot:
                     print(f'plotting {nr} images from {src}/stack')
-                    plot_arrays(src+'/stack', figuresize, cmap, nr=nr, normalize=normalize)
+                    plot_arrays(stack_path, figuresize, cmap, nr=nr, normalize=normalize)
 
                 if all_to_mip:
-                    _mip_all(src+'/stack')
+                    _mip_all(stack_path)
                     if plot:
                         print(f'plotting {nr} images from {src}/stack')
-                        plot_arrays(src+'/stack', figuresize, cmap, nr=nr, normalize=normalize)
+                        plot_arrays(stack_path, figuresize, cmap, nr=nr, normalize=normalize)
         except Exception as e:
             print(f"Error: {e}")
     
-    concatenate_and_normalize(src=src+'/stack',
+    concatenate_and_normalize(src=stack_path,
                               channels=mask_channels,
                               save_dtype=np.float32,
                               settings=settings)

spacr/measure.py

@@ -710,7 +710,7 @@ def _measure_crop_core(index, time_ls, file, settings):
         else:
             cell_mask = np.zeros_like(data[:, :, 0])
             settings['cytoplasm'] = False
-            settings['include_uninfected'] = True
+            settings['uninfected'] = True
 
         if settings['nucleus_mask_dim'] is not None:
             nucleus_mask = data[:, :, settings['nucleus_mask_dim']].astype(data_type)
@@ -762,7 +762,7 @@ def _measure_crop_core(index, time_ls, file, settings):
             cytoplasm_mask = _filter_object(cytoplasm_mask, settings['cytoplasm_min_size'])
 
         if settings['cell_mask_dim'] is not None:
-            cell_mask, nucleus_mask, pathogen_mask, cytoplasm_mask = _exclude_objects(cell_mask, nucleus_mask, pathogen_mask, cytoplasm_mask, include_uninfected=settings['include_uninfected'])
+            cell_mask, nucleus_mask, pathogen_mask, cytoplasm_mask = _exclude_objects(cell_mask, nucleus_mask, pathogen_mask, cytoplasm_mask, uninfected=settings['uninfected'])
 
         # Update data with the new masks
         if settings['cell_mask_dim'] is not None:
@@ -979,9 +979,9 @@ def measure_crop(settings):
                 #_create_database(source_folder+'/measurements/measurements.db')
             
             if settings['cell_mask_dim'] is None:
-                settings['include_uninfected'] = True
+                settings['uninfected'] = True
             if settings['pathogen_mask_dim'] is None:
-                settings['include_uninfected'] = True
+                settings['uninfected'] = True
             if settings['cell_mask_dim'] is not None and settings['pathogen_min_size'] is not None:
                 settings['cytoplasm'] = True
             elif settings['cell_mask_dim'] is not None and settings['nucleus_min_size'] is not None:

spacr/ml.py

@@ -134,7 +134,7 @@ def scale_variables(X, y):
     
     return X_scaled, y_scaled
 
-def process_model_coefficients(model, regression_type, X, y, highlight):
+def process_model_coefficients(model, regression_type, X, y, nc, pc, controls):
     """Return DataFrame of model coefficients and p-values."""
     if regression_type in ['ols', 'gls', 'wls', 'rlm', 'glm', 'mixed', 'quantile', 'logit', 'probit', 'poisson']:
         coefs = model.params
@@ -169,8 +169,8 @@ def process_model_coefficients(model, regression_type, X, y, highlight):
         coef_df['p_value'] = np.nan  # Placeholder since sklearn doesn't provide p-values
 
     coef_df['-log10(p_value)'] = -np.log10(coef_df['p_value'])
-    coef_df['highlight'] = coef_df['feature'].apply(lambda x: highlight in x)
-
+    coef_df['grna'] = coef_df['feature'].str.extract(r'\[(.*?)\]')[0]
+    coef_df['condition'] = coef_df.apply(lambda row: 'nc' if nc in row['feature'] else 'pc' if pc in row['feature'] else ('control' if row['grna'] in controls else 'other'),axis=1)
     return coef_df[~coef_df['feature'].str.contains('row|column')]
 
 def prepare_formula(dependent_variable, random_row_column_effects=False):
@@ -284,15 +284,13 @@ def check_and_clean_data(df, dependent_variable):
     df_cleaned['row'] = df['row']
     df_cleaned['column'] = df['column']
 
-    #display(df_cleaned)
-
     # Create a new column 'gene_fraction' that sums the fractions by gene within the same well
     df_cleaned['gene_fraction'] = df_cleaned.groupby(['prc', 'gene'])['fraction'].transform('sum')
 
     print("Data is ready for model fitting.")
     return df_cleaned
 
-def regression(df, csv_path, dependent_variable='predictions', regression_type=None, alpha=1.0, random_row_column_effects=False, highlight='220950', dst=None, cov_type=None):
+def regression(df, csv_path, dependent_variable='predictions', regression_type=None, alpha=1.0, random_row_column_effects=False, nc='233460', pc='220950', controls=[''], dst=None, cov_type=None, plot=False):
     from .plot import volcano_plot, plot_histogram
 
     # Generate the volcano filename
@@ -312,9 +310,7 @@ def regression(df, csv_path, dependent_variable='predictions', regression_type=N
     if regression_type is None:
         regression_type = 'ols' if is_normal else 'glm'
 
-    #display('before check_and_clean_data:',df)
     df = check_and_clean_data(df, dependent_variable)
-    #display('after check_and_clean_data:',df)
 
     # Handle mixed effects if row/column effect is treated as random
     if random_row_column_effects:
@@ -340,10 +336,10 @@ def regression(df, csv_path, dependent_variable='predictions', regression_type=N
         model = regression_model(X, y, regression_type=regression_type, groups=groups, alpha=alpha, cov_type=cov_type)
 
         # Process the model coefficients
-        coef_df = process_model_coefficients(model, regression_type, X, y, highlight)
-
-    # Plot the volcano plot
-    volcano_plot(coef_df, volcano_path)
+        coef_df = process_model_coefficients(model, regression_type, X, y, nc, pc, controls)
+    
+    if plot:
+        volcano_plot(coef_df, volcano_path)
 
     return model, coef_df
 
@@ -453,26 +449,28 @@ def perform_regression(settings):
             return df, n_gene
         else:
             return df
+        
+
     
     settings = get_perform_regression_default_settings(settings)
     count_data_df, score_data_df = _perform_regression_read_data(settings)
     results_path, results_path_gene, results_path_grna, hits_path, res_folder, csv_path = _perform_regression_set_paths(settings)
     save_settings(settings, name='regression', show=True)
+
+    if isinstance(settings['filter_value'], list):
+        filter_value = settings['filter_value']
+    else:
+        filter_value = []
+    if isinstance(settings['filter_column'], str):
+        filter_column = settings['filter_column']
     
-    score_data_df = clean_controls(score_data_df, settings['pc'], settings['nc'], settings['other'])
+    score_data_df = clean_controls(score_data_df, settings['filter_value'], settings['filter_column'])
     print(f"Dependent variable after clean_controls: {len(score_data_df)}")
 
     dependent_df, dependent_variable = process_scores(score_data_df, settings['dependent_variable'], settings['plate'], settings['min_cell_count'], settings['agg_type'], settings['transform'])
     print(f"Dependent variable after process_scores: {len(dependent_df)}")
 
-    filter_value = [settings['nc'], settings['pc']]
-
-    if settings['other'] is not None:
-        if isinstance(settings['other'], str):
-            settings['other'] = [settings['other']]
-        filter_value.extend(settings['other'])
-
-    independent_df = process_reads(count_data_df, settings['fraction_threshold'], settings['plate'], filter_column=settings['location_column'], filter_value=filter_value)
+    independent_df = process_reads(count_data_df, settings['fraction_threshold'], settings['plate'], filter_column=filter_column, filter_value=filter_value)
     independent_df, n_grna, n_gene = _count_variable_instances(independent_df, column_1='grna', column_2='gene')
 
     print(f"Independent variable after process_reads: {len(independent_df)}")
@@ -487,19 +485,30 @@ def perform_regression(settings):
     if settings['transform'] is None:
         _ = plot_plates(score_data_df, variable=dependent_variable, grouping='mean', min_max='allq', cmap='viridis', min_count=settings['min_cell_count'], dst = res_folder)                
 
-    model, coef_df = regression(merged_df, csv_path, dependent_variable, settings['regression_type'], settings['alpha'], settings['random_row_column_effects'], highlight=settings['highlight'], dst=res_folder, cov_type=settings['cov_type'])
+    model, coef_df = regression(merged_df, csv_path, dependent_variable, settings['regression_type'], settings['alpha'], settings['random_row_column_effects'], nc=settings['negative_control'], pc=settings['positive_control'], controls=settings['controls'], dst=res_folder, cov_type=settings['cov_type'])
     
     coef_df['grna'] = coef_df['feature'].apply(lambda x: re.search(r'grna\[(.*?)\]', x).group(1) if 'grna' in x else None)
     coef_df['gene'] = coef_df['feature'].apply(lambda x: re.search(r'gene\[(.*?)\]', x).group(1) if 'gene' in x else None)
     coef_df = coef_df.merge(n_grna, how='left', on='grna')
     coef_df = coef_df.merge(n_gene, how='left', on='gene')
-    display(coef_df)
 
     gene_coef_df = coef_df[coef_df['n_gene'] != None]
     grna_coef_df = coef_df[coef_df['n_grna'] != None]
     gene_coef_df = gene_coef_df.dropna(subset=['n_gene'])
     grna_coef_df = grna_coef_df.dropna(subset=['n_grna'])
     
+    if settings['controls'] is not None:
+        control_coef_df = grna_coef_df[grna_coef_df['grna'].isin(settings['controls'])]
+        mean_coef = control_coef_df['coefficient'].mean()
+        
+        if settings['threshold_method'] in ['var','variance']:
+            coef_mes = control_coef_df['coefficient'].var()
+        elif settings['threshold_method'] in ['std', 'standard_deveation']:
+            coef_mes = control_coef_df['coefficient'].std()
+        else:
+            raise ValueError(f"Unsupported threshold method {settings['threshold_method']}. Supported methods: ['var','variance','std','standard_deveation']")
+        reg_threshold = mean_coef + (settings['threshold_multiplier'] * coef_mes)
+    
     coef_df.to_csv(results_path, index=False)
     gene_coef_df.to_csv(results_path_gene, index=False)
     grna_coef_df.to_csv(results_path_grna, index=False)
@@ -509,7 +518,10 @@ def perform_regression(settings):
         
     else:
         significant = coef_df[coef_df['p_value']<= 0.05]
-        #significant = significant[significant['coefficient'] > 0.1]
+        if settings['controls'] is not None:
+            significant_high = significant[significant['coefficient'] >= reg_threshold]
+            significant_low = significant[significant['coefficient'] <= reg_threshold]
+            significant = pd.concat([significant_high, significant_low])
         significant.sort_values(by='coefficient', ascending=False, inplace=True)
         significant = significant[~significant['feature'].str.contains('row|column')]
         
@@ -530,22 +542,24 @@ def perform_regression(settings):
         grna_merged_df = merge_regression_res_with_metadata(results_path_grna, metadata_file, name=filename)
 
     if settings['toxo']:
-        
         data_path = merged_df
         data_path_gene = gene_merged_df
         data_path_grna = grna_merged_df
         base_dir = os.path.dirname(os.path.abspath(__file__))
         metadata_path = os.path.join(base_dir, 'resources', 'data', 'lopit.csv')
-
-        custom_volcano_plot(data_path, metadata_path, metadata_column='tagm_location', string_list=[settings['highlight']], point_size=50, figsize=20)
-        custom_volcano_plot(data_path_gene, metadata_path, metadata_column='tagm_location', string_list=[settings['highlight']], point_size=50, figsize=20)
-        custom_volcano_plot(data_path_grna, metadata_path, metadata_column='tagm_location', string_list=[settings['highlight']], point_size=50, figsize=20)
         
-        if len(significant) > 2:
-            metadata_path = os.path.join(base_dir, 'resources', 'data', 'toxoplasma_metadata.csv')
-            go_term_enrichment_by_column(significant, metadata_path)
+        custom_volcano_plot(data_path, metadata_path, metadata_column='tagm_location', point_size=200, figsize=20, threshold=reg_threshold, split_axis_lims=settings['split_axis_lims'])
+        #custom_volcano_plot(data_path_gene, metadata_path, metadata_column='tagm_location', point_size=50, figsize=20, threshold=reg_threshold)
+        #custom_volcano_plot(data_path_grna, metadata_path, metadata_column='tagm_location', point_size=50, figsize=20, threshold=reg_threshold)
+        
+        #if len(significant) > 2:
+        #    metadata_path = os.path.join(base_dir, 'resources', 'data', 'toxoplasma_metadata.csv')
+        #    go_term_enrichment_by_column(significant, metadata_path)
     
     print('Significant Genes')
+    grnas = significant['grna'].unique().tolist()
+    genes = significant['gene'].unique().tolist()
+    print(f"Found p<0.05 coedfficients for {len(grnas)} gRNAs and {len(genes)} genes")
     display(significant)
 
     output = {'results':coef_df,
@@ -586,7 +600,7 @@ def process_reads(csv_path, fraction_threshold, plate, filter_column=None, filte
     if isinstance(filter_value, str):
         filter_value = [filter_value]
 
-    if isinstance(filter_column, list):
+    if isinstance(filter_column, list):            
         for filter_col in filter_column:
             for value in filter_value:
                 csv_df = csv_df[csv_df[filter_col] != value]
@@ -649,16 +663,12 @@ def check_normality(data, variable_name, verbose=False):
             print(f"Normal distribution: The data for {variable_name} is not normally distributed.")
         return False
 
-def clean_controls(df,pc,nc,other):
-    if 'col' in df.columns:
-        df['column'] = df['col']
-    if nc != None:
-        df = df[~df['column'].isin([nc])]
-    if pc != None:
-        df = df[~df['column'].isin([pc])]
-    if other != None:
-        df = df[~df['column'].isin([other])]
-        print(f'Removed data from {nc, pc, other}')
+def clean_controls(df,values, column):
+    if column in df.columns:
+        if isinstance(values, list):
+            for value in values:
+                df = df[~df[column].isin([value])]
+                print(f'Removed data from {value}')
     return df
 
 def process_scores(df, dependent_variable, plate, min_cell_count=25, agg_type='mean', transform=None, regression_type='ols'):
@@ -763,7 +773,6 @@ def generate_ml_scores(settings):
             raise ValueError("The 'png_list_df' DataFrame must contain 'prcfo' and 'test' columns.")
         annotated_df = png_list_df[['prcfo', settings['annotation_column']]].set_index('prcfo')
         df = annotated_df.merge(df, left_index=True, right_index=True)
-        #display(df)
         unique_values = df[settings['annotation_column']].dropna().unique()
         if len(unique_values) == 1:
             unannotated_rows = df[df[settings['annotation_column']].isna()].index