spacr 0.3.2__py3-none-any.whl → 0.3.22__py3-none-any.whl

spacr/io.py

@@ -22,6 +22,7 @@ from torchvision.transforms import ToTensor
 import seaborn as sns 
 from nd2reader import ND2Reader
 from torchvision import transforms
+from sklearn.model_selection import train_test_split
 
 def process_non_tif_non_2D_images(folder):
     """Processes all images in the folder and splits them into grayscale channels, preserving bit depth."""
@@ -984,47 +985,6 @@ def _move_to_chan_folder(src, regex, timelapse=False, metadata_type=''):
                     shutil.move(os.path.join(src, filename), move)
     return
 
-def _merge_channels_v2(src, plot=False):
-    from .plot import plot_arrays
-    """
-    Merge the channels in the given source directory and save the merged files in a 'stack' directory.
-
-    Args:
-        src (str): The path to the source directory containing the channel folders.
-        plot (bool, optional): Whether to plot the merged arrays. Defaults to False.
-
-    Returns:
-        None
-    """
-    src = Path(src)
-    stack_dir = src / 'stack'
-    chan_dirs = [d for d in src.iterdir() if d.is_dir() and d.name in ['01', '02', '03', '04', '00', '1', '2', '3', '4','0']]
-
-    chan_dirs.sort(key=lambda x: x.name)
-    print(f'List of folders in src: {[d.name for d in chan_dirs]}. Single channel folders.')
-    start_time = time.time()
-
-    # First directory and its files
-    dir_files = list(chan_dirs[0].iterdir())
-
-    # Create the 'stack' directory if it doesn't exist
-    stack_dir.mkdir(exist_ok=True)
-    print(f'generated folder with merged arrays: {stack_dir}')
-
-    if _is_dir_empty(stack_dir):
-        with Pool(max(cpu_count() // 2, 1)) as pool:
-        #with Pool(cpu_count()) as pool:
-            merge_func = partial(_merge_file, chan_dirs, stack_dir)
-            pool.map(merge_func, dir_files)
-
-    avg_time = (time.time() - start_time) / len(dir_files)
-    print(f'Average Time: {avg_time:.3f} sec')
-
-    if plot:
-        plot_arrays(src+'/stack')
-
-    return
-
 def _merge_channels(src, plot=False):
     """
     Merge the channels in the given source directory and save the merged files in a 'stack' directory without using multiprocessing.
@@ -2384,12 +2344,8 @@ def _results_to_csv(src, df, df_well):
     wells.to_csv(wells_loc, index=True, header=True)
     cells.to_csv(cells_loc, index=True, header=True)
     return cells, wells
-    
-###################################################
-#  Classify
-###################################################
 
-def read_plot_model_stats(file_path ,save=False):
+def read_plot_model_stats(train_file_path, val_file_path ,save=False):
     
     def _plot_and_save(train_df, val_df, column='accuracy', save=False, path=None, dpi=600):
         
@@ -2418,37 +2374,19 @@ def read_plot_model_stats(file_path ,save=False):
             plt.savefig(pdf_path, format='pdf', dpi=dpi)
         else:
             plt.show()
-    # Read the CSV into a dataframe
-    df = pd.read_csv(file_path, index_col=0)
-    
-    # Split the dataframe into train and validation based on the index
-    train_df = df.filter(like='_train', axis=0).copy()
-    val_df = df.filter(like='_val', axis=0).copy()
-    
-    fldr_1 = os.path.dirname(file_path)
-    
-    train_csv_path = os.path.join(fldr_1, 'train.csv')
-    val_csv_path = os.path.join(fldr_1, 'validation.csv')
 
-    fldr_2 = os.path.dirname(fldr_1)
-    fldr_3 = os.path.dirname(fldr_2)
-    bn_1 = os.path.basename(fldr_1)
-    bn_2 = os.path.basename(fldr_2)
-    bn_3 = os.path.basename(fldr_3)
-    model_name = str(f'{bn_1}_{bn_2}_{bn_3}')
+    # Read the CSVs into DataFrames
+    train_df = pd.read_csv(train_file_path, index_col=0)
+    val_df = pd.read_csv(val_file_path, index_col=0)
 
-    # Extract epochs from index
-    train_df['epoch'] = [int(idx.split('_')[0]) for idx in train_df.index]
-    val_df['epoch'] = [int(idx.split('_')[0]) for idx in val_df.index]
-    
-    # Save dataframes to a CSV file
-    train_df.to_csv(train_csv_path)
-    val_df.to_csv(val_csv_path)
+    # Get the folder path for saving plots
+    fldr_1 = os.path.dirname(train_file_path)
     
     if save:
         # Setting the style
         sns.set(style="whitegrid")
     
+    # Plot and save the results
     _plot_and_save(train_df, val_df, column='accuracy', save=save, path=fldr_1)
     _plot_and_save(train_df, val_df, column='neg_accuracy', save=save, path=fldr_1)
     _plot_and_save(train_df, val_df, column='pos_accuracy', save=save, path=fldr_1)
@@ -2496,50 +2434,53 @@ def _save_model(model, model_type, results_df, dst, epoch, epochs, intermedeate_
     
     return model_path
 
-def _save_progress(dst, results_df, result_type='train'):
+def _save_progress(dst, train_df, validation_df):
     """
     Save the progress of the classification model.
 
     Parameters:
     dst (str): The destination directory to save the progress.
-    results_df (pandas.DataFrame): The DataFrame containing accuracy, loss, and PRAUC.
-    train_metrics_df (pandas.DataFrame): The DataFrame containing training metrics.
+    train_df (pandas.DataFrame): The DataFrame containing training stats.
+    validation_df (pandas.DataFrame): The DataFrame containing validation stats (if available).
 
     Returns:
     None
     """
+
+    def _save_df_to_csv(file_path, df):
+        """
+        Save the given DataFrame to the specified CSV file, either creating a new file or appending to an existing one.
+
+        Parameters:
+        file_path (str): The file path where the CSV will be saved.
+        df (pandas.DataFrame): The DataFrame to save.
+        """
+        if not os.path.exists(file_path):
+            with open(file_path, 'w') as f:
+                df.to_csv(f, index=True, header=True)
+                f.flush()  # Ensure data is written to the file system
+        else:
+            with open(file_path, 'a') as f:
+                df.to_csv(f, index=True, header=False)
+                f.flush()
+                
     # Save accuracy, loss, PRAUC
     os.makedirs(dst, exist_ok=True)
-    results_path = os.path.join(dst, f'{result_type}.csv')
-    if not os.path.exists(results_path):
-        results_df.to_csv(results_path, index=True, header=True, mode='w')
-    else:
-        results_df.to_csv(results_path, index=True, header=False, mode='a')
+    results_path_train = os.path.join(dst, 'train.csv')
+    results_path_validation = os.path.join(dst, 'validation.csv')
 
-    if result_type == 'train':
-        read_plot_model_stats(results_path, save=True)
-    return
+    # Save training data
+    _save_df_to_csv(results_path_train, train_df)
 
-def _save_settings(settings, src):
-    """
-    Save the settings dictionary to a CSV file.
+    # Save validation data if available
+    if validation_df is not None:
+        _save_df_to_csv(results_path_validation, validation_df)
 
-    Parameters:
-    - settings (dict): A dictionary containing the settings.
-    - src (str): The source directory where the settings file will be saved.
+        # Call read_plot_model_stats after ensuring the files are saved
+        read_plot_model_stats(results_path_train, results_path_validation, save=True)
 
-    Returns:
-    None
-    """
-    dst = os.path.join(src,'model')
-    settings_loc =  os.path.join(dst,'settings.csv')
-    os.makedirs(dst, exist_ok=True)
-    settings_df = pd.DataFrame(list(settings.items()), columns=['setting_key', 'setting_value'])
-    display(settings_df)
-    settings_df.to_csv(settings_loc, index=False)
     return
     
-    
 def _copy_missclassified(df):
     misclassified = df[df['true_label'] != df['predicted_label']]
     for _, row in misclassified.iterrows():
@@ -2869,7 +2810,8 @@ def generate_dataset(settings={}):
         all_paths = []
         for i, src in enumerate(settings['src']):
             db_path = os.path.join(src, 'measurements', 'measurements.db')
-            dst = os.path.join(src, 'datasets')
+            if i == 0:
+                dst = os.path.join(src, 'datasets')
             paths = generate_path_list_from_db(db_path, file_metadata=settings['file_metadata'])
             correct_paths(paths, src)
             all_paths.extend(paths)
@@ -2917,6 +2859,8 @@ def generate_dataset(settings={}):
 
     # Combine the temporary tar files into a final tar
     date_name = datetime.date.today().strftime('%y%m%d')
+    if len(settings['src']) > 1:
+        date_name = f"{date_name}_combined"
     if not settings['file_metadata'] is None:
         tar_name = f"{date_name}_{settings['experiment']}_{settings['file_metadata']}.tar"
     else:
@@ -2967,7 +2911,6 @@ def generate_loaders(src, mode='train', image_size=224, batch_size=32, classes=[
     - val_loaders (list): List of data loaders for validation datasets.
     """
 
-    from .io import spacrDataset
     from .utils import SelectChannels, augment_dataset
 
     chans = []
@@ -3066,10 +3009,6 @@ def generate_loaders(src, mode='train', image_size=224, batch_size=32, classes=[
 
 def generate_training_dataset(settings):
     
-    from .io import _read_and_merge_data, _read_db
-    from .utils import get_paths_from_db, annotate_conditions, save_settings
-    from .settings import set_generate_training_dataset_defaults
-    
     # Function to filter png_list_df by prcfo present in df without merging
     def filter_png_list(db_path, settings):
         tables = ['cell', 'nucleus', 'pathogen', 'cytoplasm']
@@ -3173,34 +3112,55 @@ def generate_training_dataset(settings):
                 class_paths_ls[i] = random.sample(class_paths, size)
 
         return class_paths_ls
+    
+    from .io import _read_and_merge_data, _read_db
+    from .utils import get_paths_from_db, annotate_conditions, save_settings
+    from .settings import set_generate_training_dataset_defaults
 
     # Set default settings and save
     settings = set_generate_training_dataset_defaults(settings)
     save_settings(settings, 'cv_dataset', show=True)
 
-    db_path = os.path.join(settings['src'], 'measurements', 'measurements.db')
-    dst = os.path.join(settings['src'], 'datasets', 'training')
+    class_path_list = None
 
-    # Create a new directory for training data if necessary
-    if os.path.exists(dst):
-        for i in range(1, 100000):
-            dst = os.path.join(settings['src'], 'datasets', f'training_{i}')
-            if not os.path.exists(dst):
-                print(f'Creating new directory for training: {dst}')
-                break
+    if isinstance(settings['src'], str):
+        src = [settings['src']]
 
-    # Select dataset based on dataset mode
-    if settings['dataset_mode'] == 'annotation':
-        class_paths_ls = annotation_based_selection(db_path, dst, settings)
+    for i, src in enumerate(settings['src']):
+        db_path = os.path.join(src, 'measurements', 'measurements.db')
+        
+        if len(settings['src']) > 1 and i == 0:
+            dst = os.path.join(src, 'datasets', 'training_all')
+        elif len(settings['src']) == 1:
+            dst = os.path.join(src, 'datasets', 'training')
+
+        # Create a new directory for training data if necessary
+        if os.path.exists(dst):
+            for i in range(1, 100000):
+                dst = dst + f'_{i}'
+                if not os.path.exists(dst):
+                    print(f'Creating new directory for training: {dst}')
+                    break
 
-    elif settings['dataset_mode'] == 'metadata':
-        class_paths_ls = metadata_based_selection(db_path, settings)
+        # Select dataset based on dataset mode
+        if settings['dataset_mode'] == 'annotation':
+            class_paths_ls = annotation_based_selection(db_path, dst, settings)
+
+        elif settings['dataset_mode'] == 'metadata':
+            class_paths_ls = metadata_based_selection(db_path, settings)
+
+        elif settings['dataset_mode'] == 'measurement':
+            class_paths_ls = measurement_based_selection(settings, db_path)
+        
+        if class_path_list is None:
+            class_path_list = [[] for _ in range(len(class_paths_ls))]
 
-    elif settings['dataset_mode'] == 'measurement':
-        class_paths_ls = measurement_based_selection(settings, db_path)
+        # Extend each list in class_path_list with the corresponding list from class_paths_ls
+        for idx in range(len(class_paths_ls)):
+            class_path_list[idx].extend(class_paths_ls[idx])
 
     # Generate and return training and testing directories
-    train_class_dir, test_class_dir = generate_dataset_from_lists(dst, class_data=class_paths_ls, classes=settings['classes'], test_split=settings['test_split'])
+    train_class_dir, test_class_dir = generate_dataset_from_lists(dst, class_data=class_path_list, classes=settings['classes'], test_split=settings['test_split'])
 
     return train_class_dir, test_class_dir
 
@@ -3234,7 +3194,6 @@ def training_dataset_from_annotation(db_path, dst, annotation_column='test', ann
 
 def generate_dataset_from_lists(dst, class_data, classes, test_split=0.1):
     from .utils import print_progress
-    from .deep_spacr import train_test_split
     # Make sure that the length of class_data matches the length of classes
     if len(class_data) != len(classes):
         raise ValueError("class_data and classes must have the same length.")

spacr/ml.py

@@ -1,4 +1,4 @@
-import os, shap
+import os, shap, re
 import pandas as pd
 import numpy as np
 from scipy import stats
@@ -354,75 +354,128 @@ def perform_regression(settings):
     from .settings import get_perform_regression_default_settings
     from .toxo import go_term_enrichment_by_column, custom_volcano_plot
 
-    if isinstance(settings['score_data'], list) and isinstance(settings['count_data'], list):
-        settings['plate'] = None
-        if len(settings['score_data']) == 1:
-            settings['score_data'] = settings['score_data'][0]
-        if len(settings['count_data']) == 1:
-            settings['count_data'] = settings['count_data'][0]
+    def _perform_regression_read_data(settings):
+
+        if isinstance(settings['score_data'], list) and isinstance(settings['count_data'], list):
+            settings['plate'] = None
+            if len(settings['score_data']) == 1:
+                settings['score_data'] = settings['score_data'][0]
+            if len(settings['count_data']) == 1:
+                settings['count_data'] = settings['count_data'][0]
+            else:
+                count_data_df = pd.DataFrame()
+                for i, count_data in enumerate(settings['count_data']):
+                    df = pd.read_csv(count_data)
+                    df['plate_name'] = f'plate{i+1}'
+                    if 'column' in df.columns:
+                        df['col'] = df['column']
+                    count_data_df = pd.concat([count_data_df, df])
+                    print('Count data:', len(count_data_df))
+
+                score_data_df = pd.DataFrame()
+                for i, score_data in enumerate(settings['score_data']):
+                    df = pd.read_csv(score_data)
+                    df['plate_name'] = f'plate{i+1}'
+                    if 'column' in df.columns:
+                        df['col'] = df['column']
+                    score_data_df = pd.concat([score_data_df, df])
+                    print('Score data:', len(score_data_df))
         else:
-            count_data_df = pd.DataFrame()
-            for i, count_data in enumerate(settings['count_data']):
-                df = pd.read_csv(count_data)
-                df['plate_name'] = f'plate{i+1}'
-                count_data_df = pd.concat([count_data_df, df])
-                print('Count data:', len(count_data_df))
-
-            score_data_df = pd.DataFrame()
-            for i, score_data in enumerate(settings['score_data']):
-                df = pd.read_csv(score_data)
-                df['plate_name'] = f'plate{i+1}'
-                score_data_df = pd.concat([score_data_df, df])
-                print('Score data:', len(score_data_df))
-    else:
-        count_data_df = pd.read_csv(settings['count_data'])
-        score_data_df = pd.read_csv(settings['score_data'])
-
-    reg_types = ['ols','gls','wls','rlm','glm','mixed','quantile','logit','probit','poisson','lasso','ridge']
-    if settings['regression_type'] not in reg_types:
-        print(f'Possible regression types: {reg_types}')
-        raise ValueError(f"Unsupported regression type {settings['regression_type']}")
-    
-    if settings['dependent_variable'] not in score_data_df.columns:
-        print(f'Columns in DataFrame:')
-        for col in score_data_df.columns:
-            print(col)
-        raise ValueError(f"Dependent variable {settings['dependent_variable']} not found in the DataFrame")
-
-    if isinstance(settings['count_data'], list):
-        src = os.path.dirname(settings['count_data'][0])
-        csv_path = settings['count_data'][0]
-    else:
-        src = os.path.dirname(settings['count_data'])
-        csv_path = settings['count_data']
+            count_data_df = pd.read_csv(settings['count_data'])
+            score_data_df = pd.read_csv(settings['score_data'])
 
-    settings['src'] = src
-    fldr = 'results_' + settings['regression_type']
-    if isinstance(settings['count_data'], list):
-        fldr = fldr + '_list'
+            print(f"Dependent variable: {len(score_data_df)}")
+            print(f"Independent variable: {len(count_data_df)}")
+
+        if settings['dependent_variable'] not in score_data_df.columns:
+            print(f'Columns in DataFrame:')
+            for col in score_data_df.columns:
+                print(col)
+            raise ValueError(f"Dependent variable {settings['dependent_variable']} not found in the DataFrame")
+        
+        if 'prediction_probability_class_1' in score_data_df.columns:
+            if not settings['class_1_threshold'] is None:
+                score_data_df['predictions'] = (score_data_df['prediction_probability_class_1'] >= settings['class_1_threshold']).astype(int)
+        
+        reg_types = ['ols','gls','wls','rlm','glm','mixed','quantile','logit','probit','poisson','lasso','ridge']
+        if settings['regression_type'] not in reg_types:
+            print(f'Possible regression types: {reg_types}')
+            raise ValueError(f"Unsupported regression type {settings['regression_type']}")
 
-    if settings['regression_type'] == 'quantile':
-        fldr = fldr + '_' + str(settings['alpha'])
+        return count_data_df, score_data_df
+    
+    def _perform_regression_set_paths(settings):
+        
+        if isinstance(settings['score_data'], list):
+            score_data = settings['score_data'][0]
+        else:
+            score_data = settings['score_data']
+        
+        score_source = os.path.splitext(os.path.basename(score_data))[0]
+        
+        if isinstance(settings['count_data'], list):
+            src = os.path.dirname(settings['count_data'][0])
+            csv_path = settings['count_data'][0]
+        else:
+            src = os.path.dirname(settings['count_data'])
+            csv_path = settings['count_data']
 
-    res_folder = os.path.join(src, fldr)
-    os.makedirs(res_folder, exist_ok=True)
-    results_filename = 'results.csv'
-    hits_filename = 'results_significant.csv'
-    results_path=os.path.join(res_folder, results_filename)
-    hits_path=os.path.join(res_folder, hits_filename)
+        settings['src'] = src
+        res_folder = os.path.join(src, 'results', score_source, settings['regression_type'])
+        
+        if isinstance(settings['count_data'], list):
+            res_folder = os.path.join(res_folder, 'list')
+
+        os.makedirs(res_folder, exist_ok=True)
+        results_filename = 'results.csv'
+        results_filename_gene = 'results_gene.csv'
+        results_filename_grna = 'results_grna.csv'
+        hits_filename = 'results_significant.csv'
+        results_path=os.path.join(res_folder, results_filename)
+        results_path_gene=os.path.join(res_folder, results_filename_gene)
+        results_path_grna=os.path.join(res_folder, results_filename_grna)
+        hits_path=os.path.join(res_folder, hits_filename)
+
+        return results_path, results_path_gene, results_path_grna, hits_path, res_folder, csv_path
+    
+    def _count_variable_instances(df, column_1, column_2):
+        if column_1 is not None:
+            n_grna = df[column_1].value_counts().reset_index()
+            n_grna.columns = [column_1, f'n_{column_1}']
+        if column_2 is not None:
+            n_gene = df[column_2].value_counts().reset_index()
+            n_gene.columns = [column_2, f'n_{column_2}']
+        if column_1 is not None and column_2 is not None:
+            return df, n_grna, n_gene
+        elif column_1 is not None:
+            return df, n_grna
+        elif column_2 is not None:
+            return df, n_gene
+        else:
+            return df
     
     settings = get_perform_regression_default_settings(settings)
+    count_data_df, score_data_df = _perform_regression_read_data(settings)
+    results_path, results_path_gene, results_path_grna, hits_path, res_folder, csv_path = _perform_regression_set_paths(settings)
     save_settings(settings, name='regression', show=True)
     
     score_data_df = clean_controls(score_data_df, settings['pc'], settings['nc'], settings['other'])
-
-    if 'prediction_probability_class_1' in score_data_df.columns:
-        if not settings['class_1_threshold'] is None:
-            score_data_df['predictions'] = (score_data_df['prediction_probability_class_1'] >= settings['class_1_threshold']).astype(int)
+    print(f"Dependent variable after clean_controls: {len(score_data_df)}")
 
     dependent_df, dependent_variable = process_scores(score_data_df, settings['dependent_variable'], settings['plate'], settings['min_cell_count'], settings['agg_type'], settings['transform'])
-        
-    independent_df = process_reads(count_data_df, settings['fraction_threshold'], settings['plate'])
+    print(f"Dependent variable after process_scores: {len(dependent_df)}")
+
+    filter_value = [settings['nc'], settings['pc']]
+
+    if settings['other'] is not None:
+        if isinstance(settings['other'], str):
+            settings['other'] = [settings['other']]
+        filter_value.extend(settings['other'])
+
+    independent_df = process_reads(count_data_df, settings['fraction_threshold'], settings['plate'], filter_column=settings['location_column'], filter_value=filter_value)
+    independent_df, n_grna, n_gene = _count_variable_instances(independent_df, column_1='grna', column_2='gene')
+
+    print(f"Independent variable after process_reads: {len(independent_df)}")
     
     merged_df = pd.merge(independent_df, dependent_df, on='prc')
 
@@ -436,7 +489,20 @@ def perform_regression(settings):
 
     model, coef_df = regression(merged_df, csv_path, dependent_variable, settings['regression_type'], settings['alpha'], settings['random_row_column_effects'], highlight=settings['highlight'], dst=res_folder, cov_type=settings['cov_type'])
     
+    coef_df['grna'] = coef_df['feature'].apply(lambda x: re.search(r'grna\[(.*?)\]', x).group(1) if 'grna' in x else None)
+    coef_df['gene'] = coef_df['feature'].apply(lambda x: re.search(r'gene\[(.*?)\]', x).group(1) if 'gene' in x else None)
+    coef_df = coef_df.merge(n_grna, how='left', on='grna')
+    coef_df = coef_df.merge(n_gene, how='left', on='gene')
+    display(coef_df)
+
+    gene_coef_df = coef_df[coef_df['n_gene'] != None]
+    grna_coef_df = coef_df[coef_df['n_grna'] != None]
+    gene_coef_df = gene_coef_df.dropna(subset=['n_gene'])
+    grna_coef_df = grna_coef_df.dropna(subset=['n_grna'])
+    
     coef_df.to_csv(results_path, index=False)
+    gene_coef_df.to_csv(results_path_gene, index=False)
+    grna_coef_df.to_csv(results_path_grna, index=False)
     
     if settings['regression_type'] == 'lasso':
         significant = coef_df[coef_df['coefficient'] > 0]
@@ -460,18 +526,24 @@ def perform_regression(settings):
         filename, _ = os.path.splitext(file)
         _ = merge_regression_res_with_metadata(hits_path, metadata_file, name=filename)
         merged_df = merge_regression_res_with_metadata(results_path, metadata_file, name=filename)
+        gene_merged_df = merge_regression_res_with_metadata(results_path_gene, metadata_file, name=filename)
+        grna_merged_df = merge_regression_res_with_metadata(results_path_grna, metadata_file, name=filename)
 
     if settings['toxo']:
         
         data_path = merged_df
+        data_path_gene = gene_merged_df
+        data_path_grna = grna_merged_df
         base_dir = os.path.dirname(os.path.abspath(__file__))
         metadata_path = os.path.join(base_dir, 'resources', 'data', 'lopit.csv')
 
         custom_volcano_plot(data_path, metadata_path, metadata_column='tagm_location', string_list=[settings['highlight']], point_size=50, figsize=20)
-
-        metadata_path = os.path.join(base_dir, 'resources', 'data', 'toxoplasma_metadata.csv')
-
-        go_term_enrichment_by_column(significant, metadata_path)
+        custom_volcano_plot(data_path_gene, metadata_path, metadata_column='tagm_location', string_list=[settings['highlight']], point_size=50, figsize=20)
+        custom_volcano_plot(data_path_grna, metadata_path, metadata_column='tagm_location', string_list=[settings['highlight']], point_size=50, figsize=20)
+        
+        if len(significant) > 2:
+            metadata_path = os.path.join(base_dir, 'resources', 'data', 'toxoplasma_metadata.csv')
+            go_term_enrichment_by_column(significant, metadata_path)
     
     print('Significant Genes')
     display(significant)
@@ -481,29 +553,43 @@ def perform_regression(settings):
 
     return output
 
-def process_reads(csv_path, fraction_threshold, plate):
+def process_reads(csv_path, fraction_threshold, plate, filter_column=None, filter_value=None):
 
     if isinstance(csv_path, pd.DataFrame):
         csv_df = csv_path
     else:
         # Read the CSV file into a DataFrame
         csv_df = pd.read_csv(csv_path)
-    
+
     if 'plate_name' in csv_df.columns:
         csv_df = csv_df.rename(columns={'plate_name': 'plate'})
     if 'column_name' in csv_df.columns:
         csv_df = csv_df.rename(columns={'column_name': 'column'})
+    if 'col' in csv_df.columns:
+        csv_df = csv_df.rename(columns={'col': 'column'})
     if 'row_name' in csv_df.columns:
         csv_df = csv_df.rename(columns={'row_name': 'row'})
     if 'grna_name' in csv_df.columns:
         csv_df = csv_df.rename(columns={'grna_name': 'grna'})
     if 'plate_row' in csv_df.columns:
         csv_df[['plate', 'row']] = csv_df['plate_row'].str.split('_', expand=True)
+
     if not 'plate' in csv_df.columns:
         if not plate is None:
             csv_df['plate'] = plate
         else:
             csv_df['plate'] = 'plate1'
+
+    if isinstance(filter_column, str):
+        filter_column = [filter_column]
+
+    if isinstance(filter_value, str):
+        filter_value = [filter_value]
+
+    if isinstance(filter_column, list):
+        for filter_col in filter_column:
+            for value in filter_value:
+                csv_df = csv_df[csv_df[filter_col] != value]
     
     # Ensure the necessary columns are present
     if not all(col in csv_df.columns for col in ['row','column','grna','count']):
@@ -587,7 +673,8 @@ def process_scores(df, dependent_variable, plate, min_cell_count=25, agg_type='m
     if 'col' not in df.columns:
         df['col'] = df['column']
 
-    df['prc'] = df['plate'] + '_' + df['row'] + '_' + df['col']
+    df['prc'] = df['plate'].astype(str) + '_' + df['row'].astype(str) + '_' + df['col'].astype(str)
+
     df = df[['prc', dependent_variable]]
 
     # Group by prc and calculate the mean and count of the dependent_variable