spacr 0.4.12__py3-none-any.whl → 0.4.60__py3-none-any.whl

spacr/plot.py

@@ -1658,36 +1658,36 @@ def generate_plate_heatmap(df, plate_number, variable, grouping, min_max, min_co
     num_parts = len(df['prc'].iloc[0].split('_'))
     if num_parts == 4:
         split = df['prc'].str.split('_', expand=True)
-        df['row_name'] = split[2]
+        df['rowID'] = split[2]
         df['prc'] = f"{plate_number}" + '_' + split[2] + '_' + split[3]
         
-    # Construct 'prc' based on 'plate', 'row_name', and 'column' columns
-    #df['prc'] = df['plate'].astype(str) + '_' + df['row_name'].astype(str) + '_' + df['column'].astype(str)
+    # Construct 'prc' based on 'plateID', 'rowID', and 'columnID' columns
+    #df['prc'] = df['plateID'].astype(str) + '_' + df['rowID'].astype(str) + '_' + df['columnID'].astype(str)
 
     if 'column_name' not in df.columns:
         if 'column' in df.columns:
-            df['column_name'] = df['column']
+            df['columnID'] = df['column']
         if 'column_name' in df.columns:
-            df['column_name'] = df['column_name']
+            df['columnID'] = df['column_name']
                 
-    df['plate'], df['row_name'], df['column_name'] = zip(*df['prc'].str.split('_'))
+    df['plateID'], df['rowID'], df['columnID'] = zip(*df['prc'].str.split('_'))
     
     # Filtering the dataframe based on the plate_number
-    df = df[df['plate'] == plate_number].copy()  # Create another copy after filtering
+    df = df[df['plateID'] == plate_number].copy()  # Create another copy after filtering
     
     # Ensure proper ordering
     row_order = [f'r{i}' for i in range(1, 17)]
     col_order = [f'c{i}' for i in range(1, 28)]  # Exclude c15 as per your earlier code
     
-    df['row_name'] = pd.Categorical(df['row_name'], categories=row_order, ordered=True)
-    df['column_name'] = pd.Categorical(df['column_name'], categories=col_order, ordered=True)
-    df['count'] = df.groupby(['row_name', 'column_name'])['row_name'].transform('count')
+    df['rowID'] = pd.Categorical(df['rowID'], categories=row_order, ordered=True)
+    df['columnID'] = pd.Categorical(df['columnID'], categories=col_order, ordered=True)
+    df['count'] = df.groupby(['rowID', 'columnID'])['rowID'].transform('count')
 
     if min_count > 0:
         df = df[df['count'] >= min_count]
 
     # Explicitly set observed=True to avoid FutureWarning
-    grouped = df.groupby(['row_name', 'column_name'], observed=True) # Group by row and column
+    grouped = df.groupby(['rowID', 'columnID'], observed=True) # Group by row and column
     
     if grouping == 'mean':
         plate = grouped[variable].mean().reset_index()
@@ -1699,7 +1699,7 @@ def generate_plate_heatmap(df, plate_number, variable, grouping, min_max, min_co
     else:
         raise ValueError(f"Unsupported grouping: {grouping}")
         
-    plate_map = pd.pivot_table(plate, values=variable, index='row_name', columns='column_name').fillna(0)
+    plate_map = pd.pivot_table(plate, values=variable, index='rowID', columns='columnID').fillna(0)
     
     if min_max == 'all':
         min_max = [plate_map.min().min(), plate_map.max().max()]
@@ -1842,7 +1842,7 @@ def print_mask_and_flows(stack, mask, flows, overlay=True, max_size=1000, thickn
     fig.tight_layout()
     plt.show()
     
-def plot_resize(images, resized_images, labels, resized_labels):
+def plot_resize_v1(images, resized_images, labels, resized_labels):
     # Display an example image and label before and after resizing
     fig, ax = plt.subplots(2, 2, figsize=(20, 20))
     
@@ -1866,6 +1866,48 @@ def plot_resize(images, resized_images, labels, resized_labels):
     ax[1, 1].set_title('Resized Label')
     plt.show()
     
+
+def plot_resize(images, resized_images, labels, resized_labels):
+    def prepare_image(img):
+        if img.ndim == 2:
+            return img, 'gray'
+        elif img.ndim == 3:
+            if img.shape[-1] == 1:
+                return np.squeeze(img, axis=-1), 'gray'
+            elif img.shape[-1] == 3:
+                return img, None  # RGB
+            elif img.shape[-1] == 4:
+                return img, None  # RGBA
+            else:
+                # fallback: average across channels to show as grayscale
+                return np.mean(img, axis=-1), 'gray'
+        else:
+            raise ValueError(f"Unsupported image shape: {img.shape}")
+
+    fig, ax = plt.subplots(2, 2, figsize=(20, 20))
+
+    # Original Image
+    img, cmap = prepare_image(images[0])
+    ax[0, 0].imshow(img, cmap=cmap)
+    ax[0, 0].set_title('Original Image')
+
+    # Resized Image
+    img, cmap = prepare_image(resized_images[0])
+    ax[0, 1].imshow(img, cmap=cmap)
+    ax[0, 1].set_title('Resized Image')
+
+    # Labels (assumed grayscale or single-channel)
+    lbl, cmap = prepare_image(labels[0])
+    ax[1, 0].imshow(lbl, cmap=cmap)
+    ax[1, 0].set_title('Original Label')
+
+    lbl, cmap = prepare_image(resized_labels[0])
+    ax[1, 1].imshow(lbl, cmap=cmap)
+    ax[1, 1].set_title('Resized Label')
+
+    plt.tight_layout()
+    plt.show()
+    
 def normalize_and_visualize(image, normalized_image, title=""):
     """Utility function for visualization"""
     fig, ax = plt.subplots(1, 2, figsize=(12, 6))
@@ -2529,8 +2571,8 @@ class spacrGraph:
     #    # Group by 'prc' column if representation is 'well'
     #    if self.representation == 'well':
     #        df = df.groupby(['prc', self.grouping_column])[self.data_column].agg(self.summary_func).reset_index()
-    #    if self.representation == 'plate':
-    #        df = df.groupby(['plate', self.grouping_column])[self.data_column].agg(self.summary_func).reset_index()
+    #    if self.representation == 'plateID':
+    #        df = df.groupby(['plateID', self.grouping_column])[self.data_column].agg(self.summary_func).reset_index()
     #    if self.order:
     #        df[self.grouping_column] = pd.Categorical(df[self.grouping_column], categories=self.order, ordered=True)
     #    else:
@@ -2539,8 +2581,8 @@ class spacrGraph:
   
     def preprocess_data(self):
         """
-        Preprocess the data: remove NaNs, optionally ensure 'plate' column is created,
-        then group by either 'prc', 'plate', or do no grouping at all if representation == 'object'.
+        Preprocess the data: remove NaNs, optionally ensure 'plateID' column is created,
+        then group by either 'prc', 'plateID', or do no grouping at all if representation == 'object'.
         """
         # 1) Remove NaNs in both the grouping column and each data column
         df = self.df.dropna(subset=[self.grouping_column] + self.data_column)
@@ -2555,21 +2597,21 @@ class spacrGraph:
             # Group by ['prc', grouping_column]
             group_cols = ['prc', self.grouping_column]
 
-        elif self.representation == 'plate':
-            # Make sure 'plate' exists (split from 'prc' if needed)
-            if 'plate' not in df.columns:
+        elif self.representation == 'plateID':
+            # Make sure 'plateID' exists (split from 'prc' if needed)
+            if 'plateID' not in df.columns:
                 if 'prc' in df.columns:
-                    df[['plate', 'row', 'column']] = df['prc'].str.split('_', expand=True)
+                    df[['plateID', 'rowID', 'columnID']] = df['prc'].str.split('_', expand=True)
                 else:
                     raise KeyError(
-                        "Representation is 'plate', but no 'plate' column found. "
+                        "Representation is 'plateID', but no 'plateID' column found. "
                         "Also cannot split from 'prc' because 'prc' column is missing."
                     )
-            # If the grouping column IS 'plate', only group by ['plate'] once
-            if self.grouping_column == 'plate':
-                group_cols = ['plate']
+            # If the grouping column IS 'plateID', only group by ['plateID'] once
+            if self.grouping_column == 'plateID':
+                group_cols = ['plateID']
             else:
-                group_cols = ['plate', self.grouping_column]
+                group_cols = ['plateID', self.grouping_column]
 
         else:
             raise ValueError(f"Unknown representation: {self.representation}")
@@ -3384,7 +3426,7 @@ def plot_data_from_db(settings):
         dfs.append(dft)
         
     df = pd.concat(dfs, axis=0)
-    df['prc'] = df['plate'].astype(str) + '_' + df['row_name'].astype(str) + '_' + df['column_name'].astype(str)
+    df['prc'] = df['plateID'].astype(str) + '_' + df['rowID'].astype(str) + '_' + df['columnID'].astype(str)
     
     if settings['cell_plate_metadata'] !=  None:
         df = df.dropna(subset='host_cell')
@@ -3461,19 +3503,19 @@ def plot_data_from_csv(settings):
     dfs = []
     for i, src in enumerate(srcs):
         dft = pd.read_csv(src)
-        if 'plate' not in dft.columns:
-            dft['plate'] = f"plate{i+1}"
+        if 'plateID' not in dft.columns:
+            dft['plateID'] = f"plate{i+1}"
             dft['common'] = 'spacr'
         dfs.append(dft)
 
     df = pd.concat(dfs, axis=0)
     
     if 'prc' in df.columns:
-        # Check if 'plate', 'row', and 'column' are all missing from df.columns
-        if not all(col in df.columns for col in ['plate', 'row_name', 'column_name']):
+        # Check if 'plateID', 'rowID', and 'columnID' are all missing from df.columns
+        if not all(col in df.columns for col in ['plate', 'rowID', 'columnID']):
             try:
-                # Split 'prc' into 'plate', 'row', and 'column'
-                df[['plate', 'row_name', 'column_name']] = df['prc'].str.split('_', expand=True)
+                # Split 'prc' into 'plateID', 'rowID', and 'columnID'
+                df[['plateID', 'rowID', 'columnID']] = df['prc'].str.split('_', expand=True)
             except Exception as e:
                 print(f"Could not split the prc column: {e}")
 
@@ -3827,7 +3869,7 @@ def plot_proportion_stacked_bars(settings, df, group_column, bin_column, prc_col
     pairwise_results = chi_pairwise(raw_counts, verbose=settings.get('verbose', False))
 
     # Plot based on level setting
-    if level in ['well', 'plate']:
+    if level in ['well', 'plateID']:
         # Aggregate by well for mean ± SD visualization
         well_proportions = (
             df.groupby([group_column, prc_column, bin_column])

spacr/sequencing.py

@@ -43,9 +43,9 @@ def save_unique_combinations_to_csv(unique_combinations, csv_file):
         if not existing_df.empty:
             unique_combinations = pd.concat([existing_df, unique_combinations])
             unique_combinations = unique_combinations.groupby(
-                ['row_name', 'column_name', 'grna_name'], as_index=False).sum()
+                ['rowID', 'columnID', 'grna_name'], as_index=False).sum()
 
-        unique_combinations.to_csv(csv_file, index=False)
+        unique_combinations.to_csv(csv_file, index=True)
     except Exception as e:
         print(f"Error while saving unique combinations to CSV: {e}")
 
@@ -92,7 +92,8 @@ def process_chunk(chunk_data):
     def paired_find_sequence_in_chunk_reads(r1_chunk, r2_chunk, target_sequence, offset_start, expected_end, regex):
 
         consensus_sequences, columns, grnas, rows = [], [], [], []
-
+        consensus_seq = None
+        
         for r1_lines, r2_lines in zip(r1_chunk, r2_chunk):
             _, r1_sequence, _, r1_quality = r1_lines.split('\n')
             _, r2_sequence, _, r2_quality = r2_lines.split('\n')
@@ -123,23 +124,32 @@ def process_chunk(chunk_data):
                     match = re.match(regex, consensus_seq)
                     if match:
                         consensus_sequences.append(consensus_seq)
-                        column_sequence = match.group('column')
+                        
+                        #print(f"r1_seq: {r1_seq}")
+                        #print(f"r2_seq: {r2_seq}")
+                        #print(f"consensus_sequences: {consensus_sequences}")
+                        
+                        column_sequence = match.group('columnID')
                         grna_sequence = match.group('grna')
-                        row_sequence = match.group('row_name')
+                        row_sequence = match.group('rowID')
                         columns.append(column_sequence)
                         grnas.append(grna_sequence)
                         rows.append(row_sequence)
+                        
+                        #print(f"row bc: {row_sequence} col bc: {column_sequence} grna bc: {grna_sequence}")
+                        #print(f"row bc: {rows} col bc: {columns} grna bc: {grnas}")
 
         if len(consensus_sequences) == 0:
             print(f"WARNING: No sequences matched {regex} in chunk")
             print(f"Are bacode sequences in the correct orientation?")
             print(f"Is {consensus_seq} compatible with {regex} ?")
-
-            if len(consensus_seq) >= expected_end:
-                consensus_seq_rc = reverse_complement(consensus_seq)
-                match = re.match(regex, consensus_seq_rc)
-                if match:
-                    print(f"Reverse complement of last sequence in chunk matched {regex}")
+            
+            if consensus_seq:
+                if len(consensus_seq) >= expected_end:
+                    consensus_seq_rc = reverse_complement(consensus_seq)
+                    match = re.match(regex, consensus_seq_rc)
+                    if match:
+                        print(f"Reverse complement of last sequence in chunk matched {regex}")
 
         return consensus_sequences, columns, grnas, rows
     
@@ -174,9 +184,9 @@ def process_chunk(chunk_data):
                     match = re.match(regex, consensus_seq)
                     if match:
                         consensus_sequences.append(consensus_seq)
-                        column_sequence = match.group('column')
+                        column_sequence = match.group('columnID')
                         grna_sequence = match.group('grna')
-                        row_sequence = match.group('row_name')
+                        row_sequence = match.group('rowID')
                         columns.append(column_sequence)
                         grnas.append(grna_sequence)
                         rows.append(row_sequence)
@@ -194,10 +204,10 @@ def process_chunk(chunk_data):
 
         return consensus_sequences, columns, grnas, rows
 
+    if len(chunk_data) == 10:
+        r1_chunk, r2_chunk, regex, target_sequence, offset_start, expected_end, column_csv, grna_csv, row_csv, fill_na = chunk_data
     if len(chunk_data) == 9:
-        r1_chunk, r2_chunk, regex, target_sequence, offset_start, expected_end, column_csv, grna_csv, row_csv = chunk_data
-    if len(chunk_data) == 8:
-        r1_chunk, regex, target_sequence, offset_start, expected_end, column_csv, grna_csv, row_csv = chunk_data
+        r1_chunk, regex, target_sequence, offset_start, expected_end, column_csv, grna_csv, row_csv, fill_na = chunk_data
         r2_chunk = None
 
     if r2_chunk is None:
@@ -212,9 +222,9 @@ def process_chunk(chunk_data):
     df = pd.DataFrame({
         'read': consensus_sequences,
         'column_sequence': columns,
-        'column_name': column_names,
+        'columnID': column_names,
         'row_sequence': rows,
-        'row_name': row_names,
+        'rowID': row_names,
         'grna_sequence': grnas,
         'grna_name': grna_names
     })
@@ -223,8 +233,20 @@ def process_chunk(chunk_data):
     qc_df.columns = df.columns
     qc_df.index = ["NaN_Counts"]
     qc_df['total_reads'] = len(df)
+    
+    if fill_na:
+        df2 = df.copy()
+        if 'columnID' in df2.columns:
+            df2['columnID'] = df2['columnID'].fillna(df2['column_sequence'])
+        if 'rowID' in df2.columns:
+            df2['rowID'] = df2['rowID'].fillna(df2['row_sequence'])
+        if 'grna_name' in df2.columns:
+            df2['grna_name'] = df2['grna_name'].fillna(df2['grna_sequence'])
+        
+        unique_combinations = df2.groupby(['rowID', 'columnID', 'grna_name']).size().reset_index(name='count')
+    else:
+        unique_combinations = df.groupby(['rowID', 'columnID', 'grna_name']).size().reset_index(name='count')
 
-    unique_combinations = df.groupby(['row_name', 'column_name', 'grna_name']).size().reset_index(name='count')
     return df, unique_combinations, qc_df
 
 # Function to save data from the queue
@@ -239,7 +261,7 @@ def saver_process(save_queue, hdf5_file, save_h5, unique_combinations_csv, qc_cs
         save_unique_combinations_to_csv(unique_combinations, unique_combinations_csv)
         save_qc_df_to_csv(qc_df, qc_csv_file)
 
-def paired_read_chunked_processing(r1_file, r2_file, regex, target_sequence, offset_start, expected_end, column_csv, grna_csv, row_csv, save_h5, comp_type, comp_level, hdf5_file, unique_combinations_csv, qc_csv_file, chunk_size=10000, n_jobs=None, test=False):
+def paired_read_chunked_processing(r1_file, r2_file, regex, target_sequence, offset_start, expected_end, column_csv, grna_csv, row_csv, save_h5, comp_type, comp_level, hdf5_file, unique_combinations_csv, qc_csv_file, chunk_size=10000, n_jobs=None, test=False, fill_na=False):
 
     from .utils import count_reads_in_fastq, print_progress
 
@@ -295,14 +317,12 @@ def paired_read_chunked_processing(r1_file, r2_file, regex, target_sequence, off
                 break
 
             chunk_count += 1
-            chunk_data = (r1_chunk, r2_chunk, regex, target_sequence, offset_start, expected_end, column_csv, grna_csv, row_csv)
+            chunk_data = (r1_chunk, r2_chunk, regex, target_sequence, offset_start, expected_end, column_csv, grna_csv, row_csv, fill_na)
 
-            # Process chunks in parallel
+            # Process chunks in parallel-
             result = pool.apply_async(process_chunk, (chunk_data,))
 
             df, unique_combinations, qc_df = result.get()
-
-            # Queue the results for saving
             save_queue.put((df, unique_combinations, qc_df))
 
             end_time = time.time()
@@ -323,7 +343,7 @@ def paired_read_chunked_processing(r1_file, r2_file, regex, target_sequence, off
     save_queue.put("STOP")
     save_process.join()
 
-def single_read_chunked_processing(r1_file, r2_file, regex, target_sequence, offset_start, expected_end, column_csv, grna_csv, row_csv, save_h5, comp_type, comp_level, hdf5_file, unique_combinations_csv, qc_csv_file, chunk_size=10000, n_jobs=None, test=False):
+def single_read_chunked_processing(r1_file, r2_file, regex, target_sequence, offset_start, expected_end, column_csv, grna_csv, row_csv, save_h5, comp_type, comp_level, hdf5_file, unique_combinations_csv, qc_csv_file, chunk_size=10000, n_jobs=None, test=False, fill_na=False):
 
     from .utils import count_reads_in_fastq, print_progress
 
@@ -373,10 +393,11 @@ def single_read_chunked_processing(r1_file, r2_file, regex, target_sequence, off
                 break
 
             chunk_count += 1
-            chunk_data = (r1_chunk, regex, target_sequence, offset_start, expected_end, column_csv, grna_csv, row_csv)
+            chunk_data = (r1_chunk, regex, target_sequence, offset_start, expected_end, column_csv, grna_csv, row_csv, fill_na)
 
             # Process chunks in parallel
             result = pool.apply_async(process_chunk, (chunk_data,))
+            
             df, unique_combinations, qc_df = result.get()
 
             # Queue the results for saving
@@ -414,6 +435,8 @@ def generate_barecode_mapping(settings={}):
     print(f'Using regex: {regex} to extract barcode information')
 
     samples_dict = parse_gz_files(settings['src'])
+    
+    print(samples_dict)
 
     print(f'If compression is low and save_h5 is True, saving might take longer than processing.')
     
@@ -462,7 +485,8 @@ def generate_barecode_mapping(settings={}):
                      qc_csv_file=qc_csv_file,
                      chunk_size=settings['chunk_size'],
                      n_jobs=settings['n_jobs'],
-                     test=settings['test'])
+                     test=settings['test'],
+                     fill_na=settings['fill_na'])
 
 # Function to read the CSV, compute reverse complement, and save it
 def barecodes_reverse_complement(csv_file):
@@ -489,7 +513,7 @@ def barecodes_reverse_complement(csv_file):
 
 def graph_sequencing_stats(settings):
 
-    from .utils import correct_metadata_column_names
+    from .utils import correct_metadata_column_names, correct_metadata
 
     def _plot_density(df, dependent_variable, dst=None):
         """Plot a density plot of the dependent variable."""
@@ -532,7 +556,7 @@ def graph_sequencing_stats(settings):
         # Iterate through the fraction thresholds
         for threshold in fraction_thresholds:
             filtered_df = df[df['fraction'] >= threshold]
-            unique_count = filtered_df.groupby(['plate', 'row_name', 'column'])['grna'].nunique().mean()
+            unique_count = filtered_df.groupby(['plateID', 'rowID', 'columnID'])['grna'].nunique().mean()
             results.append((threshold, unique_count))
 
         results_df = pd.DataFrame(results, columns=['fraction_threshold', 'unique_count'])
@@ -568,8 +592,19 @@ def graph_sequencing_stats(settings):
     dfs = []
     for i, count_data in enumerate(settings['count_data']):
         df = pd.read_csv(count_data)
-        df['plate'] = f'plate{i+1}'
-        df['prc'] = df['plate'].astype(str) + '_' + df['row_name'].astype(str) + '_' + df['column_name'].astype(str)
+        
+        df = correct_metadata(df)
+        
+        if 'plateID' not in df.columns:
+            df['plateID'] = f'plate{i+1}'
+            
+        display(df)
+        
+        if all(col in df.columns for col in ['plateID', 'rowID', 'columnID']):
+            df['prc'] = df['plateID'].astype(str) + '_' + df['rowID'].astype(str) + '_' + df['columnID'].astype(str)
+        else:
+            raise ValueError("The DataFrame must contain 'plateID', 'rowID', and 'columnID' columns.")
+        
         df['total_count'] = df.groupby(['prc'])['count'].transform('sum')
         df['fraction'] = df['count'] / df['total_count']
         dfs.append(df)
@@ -588,20 +623,20 @@ def graph_sequencing_stats(settings):
     # Apply the closest threshold to the DataFrame
     df = df[df['fraction'] >= closest_threshold]
 
-    # Group by 'plate', 'row_name', 'column' and compute unique counts of 'grna'
-    unique_counts = df.groupby(['plate', 'row_name', 'column'])['grna'].nunique().reset_index(name='unique_counts')
-    unique_count_mean = df.groupby(['plate', 'row_name', 'column'])['grna'].nunique().mean()
-    unique_count_std = df.groupby(['plate', 'row_name', 'column'])['grna'].nunique().std()
+    # Group by 'plateID', 'rowID', 'columnID' and compute unique counts of 'grna'
+    unique_counts = df.groupby(['plateID', 'rowID', 'columnID'])['grna'].nunique().reset_index(name='unique_counts')
+    unique_count_mean = df.groupby(['plateID', 'rowID', 'columnID'])['grna'].nunique().mean()
+    unique_count_std = df.groupby(['plateID', 'rowID', 'columnID'])['grna'].nunique().std()
 
     # Merge the unique counts back into the original DataFrame
-    df = pd.merge(df, unique_counts, on=['plate', 'row_name', 'column'], how='left')
+    df = pd.merge(df, unique_counts, on=['plateID', 'rowID', 'columnID'], how='left')
 
     print(f"unique_count mean: {unique_count_mean} std: {unique_count_std}")
     #_plot_density(df, dependent_variable='unique_counts')
     
-    has_underscore = df['row_name'].str.contains('_').any()
+    has_underscore = df['rowID'].str.contains('_').any()
     if has_underscore:
-        df['row_name'] = df['row_name'].apply(lambda x: x.split('_')[1])
+        df['rowID'] = df['rowID'].apply(lambda x: x.split('_')[1])
     
     plot_plates(df=df, variable='unique_counts', grouping='mean', min_max='allq', cmap='viridis',min_count=0, verbose=True, dst=dst)