spacr 0.4.3__tar.gz → 0.4.5__tar.gz

{spacr-0.4.3 → spacr-0.4.5}/MANIFEST.in

@@ -35,3 +35,13 @@ prune notebooks
 prune spacr/notebooks
 prune spacr/datasets
 
+# Exclude model files from distribution
+#exclude spacr/resources/models/cp/*
+#prune spacr/resources/models/cp
+
+# Exclude all notebooks (including old ones)
+prune notebooks
+prune spacr/notebooks
+prune spacr/notebooks/submodules
+prune spacr/notebooks/old
+

{spacr-0.4.3/spacr.egg-info → spacr-0.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: spacr
-Version: 0.4.3
+Version: 0.4.5
 Summary: Spatial phenotype analysis of crisp screens (SpaCr)
 Home-page: https://github.com/EinarOlafsson/spacr
 Author: Einar Birnir Olafsson

{spacr-0.4.3 → spacr-0.4.5}/setup.py

@@ -71,7 +71,7 @@ dependencies = [
 
 setup(
     name="spacr",
-    version="0.4.3",
+    version="0.4.5",
     author="Einar Birnir Olafsson",
     author_email="olafsson@med.umich.com",
     description="Spatial phenotype analysis of crisp screens (SpaCr)",

{spacr-0.4.3 → spacr-0.4.5}/spacr/core.py

@@ -13,7 +13,7 @@ def preprocess_generate_masks(settings):
     from .plot import plot_image_mask_overlay, plot_arrays
     from .utils import _pivot_counts_table, check_mask_folder, adjust_cell_masks, print_progress, save_settings, delete_intermedeate_files, format_path_for_system, normalize_src_path
     from .settings import set_default_settings_preprocess_generate_masks
-    
+        
     if 'src' in settings:
         if not isinstance(settings['src'], (str, list)):
             ValueError(f'src must be a string or a list of strings')
@@ -33,9 +33,15 @@ def preprocess_generate_masks(settings):
             
             print(f'Processing folder: {source_folder}')
             
+            source_folder = format_path_for_system(source_folder)   
+            settings['src'] = source_folder
+            src = source_folder
+            settings = set_default_settings_preprocess_generate_masks(settings)
+            
             if settings['metadata_type'] == 'auto':
-                if settings['custom_regex'] == None:
+                if settings['custom_regex'] != None:
                     try:
+                        print(f"using regex: {settings['custom_regex']}")
                         convert_separate_files_to_yokogawa(folder=source_folder, regex=settings['custom_regex'])
                     except:
                         try:
@@ -52,17 +58,12 @@ def preprocess_generate_masks(settings):
                         print(f'Error: {e}')
                         return
             
-            source_folder = format_path_for_system(source_folder)            
-            settings['src'] = source_folder
-            src = source_folder
-            settings = set_default_settings_preprocess_generate_masks(settings)
-            
             if settings['cell_channel'] is None and settings['nucleus_channel'] is None and settings['pathogen_channel'] is None:
                 print(f'Error: At least one of cell_channel, nucleus_channel or pathogen_channel must be defined')
                 return
-
+            
             save_settings(settings, name='gen_mask_settings')
-
+            
             if not settings['pathogen_channel'] is None:
                 custom_model_ls = ['toxo_pv_lumen','toxo_cyto']
                 if settings['pathogen_model'] not in custom_model_ls:
@@ -82,7 +83,7 @@ def preprocess_generate_masks(settings):
                 settings_df = pd.DataFrame(list(settings.items()), columns=['setting_key', 'setting_value'])
                 settings_df['setting_value'] = settings_df['setting_value'].apply(str)
                 display(settings_df)
-
+                
             if settings['test_mode']:
                 print(f'Starting Test mode ...')
 

{spacr-0.4.3 → spacr-0.4.5}/spacr/io.py

@@ -647,7 +647,7 @@ def load_images_from_paths(images_by_key):
     
     return images_dict
 
-#@log_function_call
+#@log_function_call 
 def _rename_and_organize_image_files(src, regex, batch_size=100, pick_slice=False, skip_mode='01', metadata_type='', img_format='.tif'):
     """
     Convert z-stack images to maximum intensity projection (MIP) images.
@@ -664,13 +664,16 @@ def _rename_and_organize_image_files(src, regex, batch_size=100, pick_slice=Fals
         None
     """
     
+    if isinstance(img_format, str):
+        img_format = [img_format]
+    
     from .utils import _extract_filename_metadata, print_progress
     
     regular_expression = re.compile(regex)
     stack_path = os.path.join(src, 'stack')
     files_processed = 0
     if not os.path.exists(stack_path) or (os.path.isdir(stack_path) and len(os.listdir(stack_path)) == 0):
-        all_filenames = [filename for filename in os.listdir(src) if filename.endswith(img_format)]
+        all_filenames = [filename for filename in os.listdir(src) if any(filename.endswith(ext) for ext in img_format)]
         print(f'All files: {len(all_filenames)} in {src}')
         time_ls = []
         image_paths_by_key = _extract_filename_metadata(all_filenames, src, regular_expression, metadata_type, pick_slice, skip_mode)
@@ -729,11 +732,11 @@ def _rename_and_organize_image_files(src, regex, batch_size=100, pick_slice=Fals
             images_by_key.clear()
 
         # Move original images to a new directory
-        valid_exts = [img_format]
         newpath = os.path.join(src, 'orig')
         os.makedirs(newpath, exist_ok=True)
         for filename in os.listdir(src):
-            if os.path.splitext(filename)[1] in valid_exts:
+            #print(f"{filename}: {os.path.splitext(filename)[1]}")
+            if os.path.splitext(filename)[1] in img_format:
                 move = os.path.join(newpath, filename)
                 if os.path.exists(move):
                     print(f'WARNING: A file with the same name already exists at location {move}')
@@ -1571,7 +1574,7 @@ def preprocess_img_data(settings):
     Returns:
         None
     """
-
+    
     src = settings['src']
     valid_ext = ['tif', 'tiff', 'png', 'jpeg']
     files = os.listdir(src)
@@ -1599,11 +1602,11 @@ def preprocess_img_data(settings):
         
     mask_channels = [settings['nucleus_channel'], settings['cell_channel'], settings['pathogen_channel']]
     backgrounds = [settings['nucleus_background'], settings['cell_background'], settings['pathogen_background']]
-
+    
     settings, metadata_type, custom_regex, nr, plot, batch_size, timelapse, lower_percentile, randomize, all_to_mip, pick_slice, skip_mode, cmap, figuresize, normalize, save_dtype, test_mode, test_images, random_test = set_default_settings_preprocess_img_data(settings)
-
+    
     regex = _get_regex(metadata_type, img_format, custom_regex)
-
+    
     if test_mode:
 
         print(f'Running spacr in test mode')
@@ -1612,6 +1615,8 @@ def preprocess_img_data(settings):
             os.rmdir(os.path.join(src, 'test'))
             print(f"Deleted test directory: {os.path.join(src, 'test')}")
         except OSError as e:
+            print(f"Error deleting test directory: {e}")
+            print(f"Delete manually before running test mode")
             pass
 
         src = _run_test_mode(settings['src'], regex, timelapse, test_images, random_test)
@@ -1628,6 +1633,7 @@ def preprocess_img_data(settings):
                 if timelapse:
                     _move_to_chan_folder(src, regex, timelapse, metadata_type)
                 else:
+                    img_format = ['.tif', '.tiff', '.png', '.jpg', '.jpeg', '.bmp', '.nd2', '.czi', '.lif']
                     _rename_and_organize_image_files(src, regex, batch_size, pick_slice, skip_mode, metadata_type, img_format)
                     
                     #Make sure no batches will be of only one image
@@ -3105,6 +3111,7 @@ def generate_dataset_from_lists(dst, class_data, classes, test_split=0.1):
     return os.path.join(dst, 'train'), os.path.join(dst, 'test')
 
 def convert_separate_files_to_yokogawa(folder, regex):
+    
     ROWS = "ABCDEFGHIJKLMNOP"
     COLS = [f"{i:02d}" for i in range(1, 25)]
     WELLS = [f"{r}{c}" for r in ROWS for c in COLS]
@@ -3121,13 +3128,13 @@ def convert_separate_files_to_yokogawa(folder, regex):
 
     pattern = re.compile(regex, re.I)
 
-    files_by_fov = {}
+    files_by_region = {}
     rename_log = []
     csv_path = os.path.join(folder, "rename_log.csv")
-    used_wells = set(os.listdir(folder))
-    fov_to_well = {}
+    used_wells = set()
+    region_to_well = {}
 
-    # Group files by FOV
+    # Group files by (plateID, wellID, fieldID, timeID, chanID)
     for file in os.listdir(folder):
         match = pattern.match(file)
         if not match:
@@ -3136,37 +3143,63 @@ def convert_separate_files_to_yokogawa(folder, regex):
 
         meta = match.groupdict()
 
-        # Extract required metadata
-        fov = meta['fov']
-        channel = int(meta['channel'])
+        # Mandatory metadata
+        if 'wellID' not in meta or meta['wellID'] is None:
+            print(f"Skipping {file}: missing mandatory wellID.")
+            continue
+        wellID = meta['wellID']
 
         # Optional metadata with defaults
-        time = int(meta.get('time', 1))
-        z_slice = int(meta.get('slice', 1))
-
-        files_by_fov.setdefault(fov, []).append((file, channel, time, z_slice))
-
-    # Assign wells per FOV
-    for fov, file_list in files_by_fov.items():
-        if fov not in fov_to_well:
-            fov_to_well[fov] = _get_next_well(used_wells)
-            used_wells.add(fov_to_well[fov])
-
-        well = fov_to_well[fov]
+        plateID = meta.get('plateID', '1') or '1'
+        fieldID = meta.get('fieldID', '1') or '1'
+        timeID = int(meta.get('timeID', 1) or 1)
+        chanID = int(meta.get('chanID', 1) or 1)
+        sliceID = meta.get('sliceID')
+        sliceID = int(sliceID) if sliceID is not None else None
+
+        region_key = (plateID, wellID, fieldID, timeID, chanID)
+
+        files_by_region.setdefault(region_key, []).append((file, sliceID))
+
+    # Assign wells and process files per region
+    for region, file_list in files_by_region.items():
+        if region[:3] not in region_to_well:
+            next_well = _get_next_well(used_wells)
+            region_to_well[region[:3]] = next_well
+            used_wells.add(next_well)
+
+        assigned_well = region_to_well[region[:3]]
+        plateID, wellID, fieldID, timeID, chanID = region
+
+        # Check if multiple slices exist and are meaningful
+        slice_ids = [sid for _, sid in file_list if sid is not None]
+        unique_slices = set(slice_ids)
+
+        images = []
+        for filename, _ in sorted(file_list, key=lambda x: x[1] or 1):
+            img = tifffile.imread(os.path.join(folder, filename))
+            images.append(img)
+
+        # Perform MIP only if multiple unique slices are present
+        if len(unique_slices) > 1:
+            img_to_save = np.max(np.stack(images), axis=0)
+        else:
+            img_to_save = images[0]
 
-        for original_file, channel, time, z_slice in file_list:
-            img = tifffile.imread(os.path.join(folder, original_file))
-            dtype = img.dtype
+        dtype = img_to_save.dtype
 
-            filename = f"{well}_T{time:04d}F001L01C{channel:02d}Z{z_slice:03d}.tif"
-            filepath = os.path.join(folder, filename)
-            tifffile.imwrite(filepath, img.astype(dtype))
+        new_filename = f"{assigned_well}_T{timeID:04d}F{int(fieldID):03d}L01C{chanID:02d}.tif"
+        new_filepath = os.path.join(folder, new_filename)
+        tifffile.imwrite(new_filepath, img_to_save.astype(dtype))
 
-            rename_log.append({"Original File": original_file, "Renamed TIFF": filename})
+        # Log original filenames involved in MIP or single file rename
+        original_files = ";".join(f[0] for f in file_list)
+        rename_log.append({"Original File(s)": original_files, "Renamed TIFF": new_filename})
 
     pd.DataFrame(rename_log).to_csv(csv_path, index=False)
     print(f"Processing complete. Files saved in {folder} and rename log saved as {csv_path}.")
 
+
 def convert_to_yokogawa(folder):
     """
     Detects file type in the folder and converts them

{spacr-0.4.3 → spacr-0.4.5}/spacr/settings.py

@@ -1213,16 +1213,25 @@ def generate_fields(variables, scrollable_frame):
         "black_background": "(bool) - Whether to use a black background for plots.",
         "calculate_correlation": "(bool) - Whether to calculate correlations between features.",
         "cell_CP_prob": "(float) - The cellpose probability threshold for the cell channel. This will be used in cell segmentation.",
+        "nucleus_CP_prob": "(float) - The cellpose probability threshold for the nucleus channel. This will be used in cell segmentation.",
+        "pathogen_CP_prob": "(float) - The cellpose probability threshold for the pathogen channel. This will be used in cell segmentation.",
         "cell_FT": "(float) - The flow threshold for cell objects. This will be used to segment the cells.",
-        "cell_background": "(float) - The background intensity for the cell channel. This will be used to remove background noise.",
+        "nucleus_FT": "(float) - The flow threshold for nucleus objects. This will be used to segment the cells.",
+        "pathogen_FT": "(float) - The flow threshold for pathogen objects. This will be used to segment the cells.",
+        "cell_background": "(int) - The background intensity for the cell channel. This will be used to remove background noise.",
+        "nucleus_background": "(int) - The background intensity for the nucleus channel. This will be used to remove background noise.",
+        "pathogen_background": "(int) - The background intensity for the pathogen channel. This will be used to remove background noise.",
         "cell_chann_dim": "(int) - Dimension of the channel to use for cell segmentation.",
-        "cell_channel": "(int) - The channel to use for the cell. If None, the cell will not be segmented.",
+        "cell_channel": "(int) - The channel to use for generatin cell masks. If None, cell masks will not be generated.",
+        "nucleus_channel": "(int) - The channel to use for generatin nucleus masks. If None, nucleus masks will not be generated.",
+        "pathogen_channel": "(int) - The channel to use for generatin pathogen masks. If None, pathogen masks will not be generated.",
         "cell_intensity_range": "(list) - Intensity range for cell segmentation.",
         "cell_loc": "(list) - The locations of the cell types in the images.",
-        "cell_mask_dim": "(int) - The dimension of the array the cell mask is saved in.",
+        "cell_mask_dim": "(int) - The dimension of the array the cell mask is saved in (array order:channels,cell, nucleus, pathogen, cytoplasm) array starts at dimension 0.",
+        "nucleus_mask_dim": "(int) - The dimension of the array the nucleus mask is saved in (array order:channels,cell, nucleus, pathogen, cytoplasm) array starts at dimension 0.",
         "cell_min_size": "(int) - The minimum size of cell objects in pixels^2.",
         "cell_plate_metadata": "(str) - Metadata for the cell plate.",
-        "cell_Signal_to_noise": "(float) - The signal-to-noise ratio for the cell channel. This will be used to determine the range of intensities to normalize images to for cell segmentation.",
+        "cell_Signal_to_noise": "(int) - The signal-to-noise ratio for the cell channel. This will be used to determine the range of intensities to normalize images to for cell segmentation.",
         "cell_size_range": "(list) - Size range for cell segmentation.",
         "cell_types": "(list) - Types of cells to include in the analysis.",
         "cells": "(list of lists) - The cell types to include in the analysis.",
@@ -1240,10 +1249,13 @@ def generate_fields(variables, scrollable_frame):
         "CP_prob": "(float) - Cellpose probability threshold for segmentation.",
         "crop_mode": "(str) - Mode to use for cropping images (cell, nucleus, pathogen, cytoplasm).",
         "custom_model": "(str) - Path to a custom Cellpose model.",
-        "custom_regex": "(str) - Custom regex pattern to extract metadata from the image names. This will only be used if 'custom' is selected for 'metadata_type'.",
+        "custom_regex": "(str) - Custom regex pattern to extract metadata from the image names. This will only be used if 'custom' or 'auto' is selected for 'metadata_type'.",
         "cytoplasm": "(bool) - Whether to segment the cytoplasm (Cell - Nucleus + Pathogen).",
         "cytoplasm_min_size": "(int) - The minimum size of cytoplasm objects in pixels^2.",
+        "nucleus_min_size": "(int) - The minimum size of nucleus objects in pixels^2.",
+        "normalize_by": "(str) - Normalize cropped png images by png or by field of view.",
         "dependent_variable": "(str) - The dependent variable for the regression analysis.",
+        "delete_intermediate": "(bool) - Delete intermediate folders (stack, channel, norm_channel_stack).",
         "diameter": "(float) - Diameter of the objects to segment.",
         "dialate_png_ratios": "(list) - The ratios to use for dilating the PNG images. This will determine the amount of dilation applied to the images before cropping.",
         "dialate_pngs": "(bool) - Whether to dilate the PNG images before saving.",
@@ -1291,7 +1303,7 @@ def generate_fields(variables, scrollable_frame):
         "manders_thresholds": "(list) - Thresholds for Manders' coefficients.",
         "mask": "(bool) - Whether to generate masks for the segmented objects. If True, masks will be generated for the nucleus, cell, and pathogen.",
         "measurement": "(str) - The measurement to use for the analysis.",
-        "metadata_type": "(str) - Type of metadata to expect in the images. This will determine how the images are processed. If 'custom' is selected, you can provide a custom regex pattern to extract metadata from the image names.",
+        "metadata_type": "(str) - Type of metadata to expect in the images. If 'custom' is selected, you can provide a custom regex pattern to extract metadata from the image names. auto will attempt to automatically extract metadata from the image names. cellvoyager and cq1 will use the default metadata extraction for CellVoyager and CQ1 images.",
         "metadata_types": "(list) - Types of metadata to include in the analysis.",
         "merge_edge_pathogen_cells": "(bool) - Whether to merge cells that share pathogen objects.",
         "merge_pathogens": "(bool) - Whether to merge pathogen objects that share more than 75 percent of their perimeter.",
@@ -1312,7 +1324,8 @@ def generate_fields(variables, scrollable_frame):
         "n_jobs": "(int) - The number of n_jobs to use for processing the images. This will determine how many images are processed in parallel. Increase to speed up processing.",
         "n_neighbors": "(int) - Number of neighbors for UMAP.",
         "n_repeats": "(int) - Number of repeats for the pathogen plate.",
-        "pathogen_Signal_to_noise": "(float) - The signal-to-noise ratio for the pathogen channel. This will be used to determine the range of intensities to normalize images to for pathogen segmentation.",
+        "pathogen_Signal_to_noise": "(int) - The signal-to-noise ratio for the pathogen channel. This will be used to determine the range of intensities to normalize images to for pathogen segmentation.",
+        "nucleus_Signal_to_noise": "(int) - The signal-to-noise ratio for the nucleus channel. This will be used to determine the range of intensities to normalize images to for nucleus segmentation.",
         "pathogen_size_range": "(list) - Size range for pathogen segmentation.",
         "pathogen_types": "(list) - Types of pathogens to include in the analysis.",
         "pc": "(str) - Positive control identifier.",
@@ -1359,10 +1372,11 @@ def generate_fields(variables, scrollable_frame):
         "save_measurements": "(bool) - Whether to save the measurements to disk.",
         "save_png": "(bool) - Whether to save the segmented objects as PNG images.",
         "schedule": "(str) - Schedule for processing the data.",
-        "Signal_to_noise": "(float) - Signal-to-noise ratio for the images.",
+        "Signal_to_noise": "(int) - Signal-to-noise ratio for the images.",
         "skip_mode": "(str) - The mode to use for skipping images. This will determine how to handle images that cannot be processed.",
         "smooth_lines": "(bool) - Whether to smooth lines in the plots.",
         "src": "(str, path) - Path to source directory.",
+        "segmentation_mode": "(str) - Algorithm to use for segmentation (cellpose or mediar).",
         "target": "(str) - Target variable for the analysis.",
         "target_height": "(int) - Target height for resizing the images.",
         "target_intensity_min": "(float) - Minimum intensity for the target objects.",
@@ -1412,7 +1426,7 @@ def generate_fields(variables, scrollable_frame):
         "masks": "(bool) - Whether to generate masks for the segmented objects.",
         "timelapse": "(bool) - Whether to analyze images as a timelapse.",
         "pathogen_min_size": "(int) - The minimum size of pathogen objects in pixels^2.",
-        "pathogen_mask_dim": "(int) - The dimension of the array the pathogen mask is saved in.",
+        "pathogen_mask_dim": "(int) - The dimension of the array the pathogen mask is saved in (array order:channels,cell, nucleus, pathogen, cytoplasm) array starts at dimension 0.",
         "use_bounding_box": "(bool) - Whether to use the bounding box for cropping the images.",
         "plot_points": "(bool) - Whether to plot scatterplot points.",
         "embedding_by_controls": "(bool) - Use the controlls to greate the embedding, then apply this embedding to all of the data.",
@@ -1442,6 +1456,7 @@ def generate_fields(variables, scrollable_frame):
         "shuffle": "(bool) - Shuffle the dataset bufore generating the activation maps",
         "correlation": "(bool) - Calculate correlation between image channels and activation maps. Data is saved to .db.",
         "normalize_input": "(bool) - Normalize the input images before passing them to the model.",
+        "normalize_plots": "(bool) - Normalize images before plotting.",
     }
     
     for key, (var_type, options, default_value) in variables.items():

{spacr-0.4.3 → spacr-0.4.5}/spacr/utils.py

@@ -321,21 +321,24 @@ def load_settings(csv_file_path, show=False, setting_key='setting_key', setting_
 
 def save_settings(settings, name='settings', show=False):
     
-    settings_df = pd.DataFrame(list(settings.items()), columns=['Key', 'Value'])
-    if show:
-        display(settings_df)
+    settings_2 = settings.copy()
     
-    if isinstance(settings['src'], list):
-        src = settings['src'][0]
+    if isinstance(settings_2['src'], list):
+        src = settings_2['src'][0]
         name = f"{name}_list"
     else:
-        src = settings['src']
+        src = settings_2['src']
         
-    if 'test_mode' in settings.keys():
-        settings['test_mode'] = False
+    if 'test_mode' in settings_2.keys():
+        settings_2['test_mode'] = False
         
-        if 'plot' in settings.keys():
-            settings['plot'] = False
+        if 'plot' in settings_2.keys():
+            settings_2['plot'] = False
+            
+    settings_df = pd.DataFrame(list(settings_2.items()), columns=['Key', 'Value'])
+    
+    if show:
+        display(settings_df)
 
     settings_csv = os.path.join(src,'settings',f'{name}.csv')
     os.makedirs(os.path.join(src,'settings'), exist_ok=True)
@@ -554,7 +557,7 @@ def _get_cellpose_batch_size():
 def _extract_filename_metadata(filenames, src, regular_expression, metadata_type='cellvoyager', pick_slice=False, skip_mode='01'):
     
     images_by_key = defaultdict(list)
-    
+
     for filename in filenames:
         match = regular_expression.match(filename)
         if match:
@@ -597,7 +600,7 @@ def _extract_filename_metadata(filenames, src, regular_expression, metadata_type
             except IndexError:
                 print(f"Could not extract information from filename {filename} using provided regex")
         else:
-            print(f"Filename {filename} did not match provided regex")
+            print(f"Filename {filename} did not match provided regex: {regular_expression}")
             continue
         
     return images_by_key
@@ -3144,7 +3147,7 @@ def _run_test_mode(src, regex, timelapse=False, test_images=10, random_test=True
 
     if os.path.exists(os.path.join(src, 'orig')):
         src = os.path.join(src, 'orig')
-    
+        
     all_filenames = [filename for filename in os.listdir(src) if regular_expression.match(filename)]
     print(f'Found {len(all_filenames)} files')
     images_by_set = defaultdict(list)

{spacr-0.4.3 → spacr-0.4.5/spacr.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: spacr
-Version: 0.4.3
+Version: 0.4.5
 Summary: Spatial phenotype analysis of crisp screens (SpaCr)
 Home-page: https://github.com/EinarOlafsson/spacr
 Author: Einar Birnir Olafsson