spacr 0.2.81__tar.gz → 0.3.1__tar.gz

spacr-0.3.1/MANIFEST.in

@@ -0,0 +1,5 @@
+recursive-include spacr/resources/models/cp *
+recursive-include spacr/resources/icons *
+recursive-include spacr/resources/font *
+recursive-include spacr/resources/MEDIAR *
+recursive-include spacr/resources/MEDIAR_weights *

{spacr-0.2.81/spacr.egg-info → spacr-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: spacr
-Version: 0.2.81
+Version: 0.3.1
 Summary: Spatial phenotype analysis of crisp screens (SpaCr)
 Home-page: https://github.com/EinarOlafsson/spacr
 Author: Einar Birnir Olafsson
@@ -50,6 +50,14 @@ Requires-Dist: rapidfuzz<4.0,>=3.9
 Requires-Dist: keyring<16.0,>=15.1
 Requires-Dist: screeninfo<1.0,>=0.8.1
 Requires-Dist: ipykernel
+Requires-Dist: gdown
+Requires-Dist: fastremap>=1.14.1
+Requires-Dist: monai>=1.3.0
+Requires-Dist: pytz>=2023.3.post1
+Requires-Dist: segmentation_models_pytorch>=0.3.3
+Requires-Dist: tifffile>=2023.4.12
+Requires-Dist: tqdm>=4.65.0
+Requires-Dist: wandb>=0.16.2
 Requires-Dist: huggingface-hub<0.25,>=0.24.0
 Provides-Extra: dev
 Requires-Dist: pytest<3.11,>=3.9; extra == "dev"

{spacr-0.2.81 → spacr-0.3.1}/setup.py

@@ -47,12 +47,20 @@ dependencies = [
     'keyring>=15.1, <16.0',
     'screeninfo>=0.8.1,<1.0',
     'ipykernel',
+    'gdown',
+    'fastremap>=1.14.1',
+    'monai>=1.3.0',
+    'pytz>=2023.3.post1',
+    'segmentation_models_pytorch>=0.3.3',
+    'tifffile>=2023.4.12',
+    'tqdm>=4.65.0',
+    'wandb>=0.16.2',
     'huggingface-hub>=0.24.0,<0.25'
 ]
 
 setup(
     name="spacr",
-    version="0.2.81",
+    version="0.3.01",
     author="Einar Birnir Olafsson",
     author_email="olafsson@med.umich.com",
     description="Spatial phenotype analysis of crisp screens (SpaCr)",
@@ -60,7 +68,7 @@ setup(
     url="https://github.com/EinarOlafsson/spacr",
     packages=find_packages(exclude=["tests.*", "tests"]),
     include_package_data=True,
-    package_data={'spacr': ['resources/models/cp/*', 'resources/icons/*', 'resources/font/**/*'],},
+    package_data={'spacr': ['resources/models/cp/*', 'resources/icons/*', 'resources/font/**/*', 'resources/MEDIAR/**/*', 'resources/MEDIAR_weights/*', 'resources/images/*'],},
     install_requires=dependencies,
     entry_points={
         'console_scripts': [
@@ -100,4 +108,5 @@ for dep in deps:
     try:
         subprocess.run(['pip', 'install', dep], check=True)
     except subprocess.CalledProcessError:
-        pass
+        pass
+

{spacr-0.2.81 → spacr-0.3.1}/spacr/__init__.py

@@ -23,9 +23,9 @@ from . import app_measure
 from . import app_classify
 from . import app_sequencing
 from . import app_umap
+from . import mediar
 from . import logger
 
-
 __all__ = [
     "core",
     "io",
@@ -48,6 +48,7 @@ __all__ = [
     "app_classify",
     "app_sequencing",
     "app_umap",
+    "mediar",
     "logger"
 ]
 

{spacr-0.2.81 → spacr-0.3.1}/spacr/core.py

@@ -50,8 +50,6 @@ import random
 from PIL import Image
 from torchvision.transforms import ToTensor
 
-
-
 def analyze_plaques(folder):
     summary_data = []
     details_data = []
@@ -1674,7 +1672,10 @@ def preprocess_generate_masks(src, settings={}):
             time_ls=[]
             if check_mask_folder(src, 'cell_mask_stack'):
                 start = time.time()
-                generate_cellpose_masks(mask_src, settings, 'cell')
+                if settings['segmentation_mode'] == 'cellpose':
+                    generate_cellpose_masks(mask_src, settings, 'cell')
+                elif settings['segmentation_mode'] == 'mediar':
+                    generate_mediar_masks(mask_src, settings, 'cell')
                 stop = time.time()
                 duration = (stop - start)
                 time_ls.append(duration)
@@ -1685,7 +1686,10 @@ def preprocess_generate_masks(src, settings={}):
             time_ls=[]
             if check_mask_folder(src, 'nucleus_mask_stack'):
                 start = time.time()
-                generate_cellpose_masks(mask_src, settings, 'nucleus')
+                if settings['segmentation_mode'] == 'cellpose':
+                    generate_cellpose_masks(mask_src, settings, 'nucleus')
+                elif settings['segmentation_mode'] == 'mediar':
+                    generate_mediar_masks(mask_src, settings, 'nucleus')
                 stop = time.time()
                 duration = (stop - start)
                 time_ls.append(duration)
@@ -1696,7 +1700,10 @@ def preprocess_generate_masks(src, settings={}):
             time_ls=[]
             if check_mask_folder(src, 'pathogen_mask_stack'):
                 start = time.time()
-                generate_cellpose_masks(mask_src, settings, 'pathogen')
+                if settings['segmentation_mode'] == 'cellpose':
+                    generate_cellpose_masks(mask_src, settings, 'pathogen')
+                elif settings['segmentation_mode'] == 'mediar':
+                    generate_mediar_masks(mask_src, settings, 'pathogen')
                 stop = time.time()
                 duration = (stop - start)
                 time_ls.append(duration)
@@ -1750,7 +1757,7 @@ def preprocess_generate_masks(src, settings={}):
                             files_processed = i+1
                             files_to_process = settings['examples_to_plot']
                             print_progress(files_processed, files_to_process, n_jobs=1, time_ls=time_ls, batch_size=None, operation_type="Plot mask outlines")
-                            print("Successfully completed run")
+                            
                 except Exception as e:
                     print(f'Failed to plot image mask overly. Error: {e}')
             else:
@@ -1898,7 +1905,7 @@ def all_elements_match(list1, list2):
     # Check if all elements in list1 are in list2
     return all(element in list2 for element in list1)
 
-def prepare_batch_for_cellpose(batch):
+def prepare_batch_for_segmentation(batch):
     # Ensure the batch is of dtype float32
     if batch.dtype != np.float32:
         batch = batch.astype(np.float32)
@@ -2021,7 +2028,7 @@ def generate_cellpose_masks(src, settings, object_type):
             if batch.size == 0:
                 continue
             
-            batch = prepare_batch_for_cellpose(batch)
+            batch = prepare_batch_for_segmentation(batch)
 
             if timelapse:
                 movie_path = os.path.join(os.path.dirname(src), 'movies')
@@ -2180,7 +2187,7 @@ def generate_masks_from_imgs(src, model, model_name, batch_size, diameter, cellp
         image_files = all_image_files[i:i+batch_size]
 
         if normalize:
-            images, _, image_names, _ = _load_normalized_images_and_labels(image_files, None, channels, percentiles,  circular, invert, plot, remove_background, background, Signal_to_noise)
+            images, _, image_names, _, orig_dims = _load_normalized_images_and_labels(image_files, None, channels, percentiles,  circular, invert, plot, remove_background, background, Signal_to_noise, target_height, target_width)
             images = [np.squeeze(img) if img.shape[-1] == 1 else img for img in images]
             orig_dims = [(image.shape[0], image.shape[1]) for image in images]
         else:
@@ -2300,7 +2307,7 @@ def compare_cellpose_masks(src, verbose=False, processes=None, save=True):
     from .io import _read_mask
 
     dirs = [os.path.join(src, d) for d in os.listdir(src) if os.path.isdir(os.path.join(src, d)) and d != 'results']
-    dirs.sort()  # Optional: sort directories if needed
+    dirs.sort()
     conditions = [os.path.basename(d) for d in dirs]
 
     # Get common files in all directories
@@ -2316,7 +2323,7 @@ def compare_cellpose_masks(src, verbose=False, processes=None, save=True):
 
     # Filter out None results (from skipped files)
     results = [res for res in results if res is not None]
-    #print(results)
+    print(results)
     if verbose:
         for result in results:
             filename = result['filename']
@@ -3102,4 +3109,92 @@ def reducer_hyperparameter_search(settings={}, reduction_params=None, dbscan_par
     else:
         plt.show()
 
-    return
+    return
+
+def generate_mediar_masks(src, settings, object_type):
+    """
+    Generates masks using the MEDIARPredictor.
+    
+    :param src: Source folder containing images or npz files.
+    :param settings: Dictionary of settings for generating masks.
+    :param object_type: Type of object to detect (e.g., 'cell', 'nucleus', etc.).
+    """
+    from .mediar import MEDIARPredictor
+    from .io import _create_database, _save_object_counts_to_database
+    from .plot import plot_masks
+    from .settings import set_default_settings_preprocess_generate_masks, _get_object_settings
+
+    # Clear CUDA cache and check if CUDA is available
+    gc.collect()
+    if not torch.cuda.is_available():
+        print(f'Torch CUDA is not available, using CPU')
+
+    # Preprocess settings
+    settings = set_default_settings_preprocess_generate_masks(src, settings)
+
+    if settings['verbose']:
+        settings_df = pd.DataFrame(list(settings.items()), columns=['setting_key', 'setting_value'])
+        settings_df['setting_value'] = settings_df['setting_value'].apply(str)
+        display(settings_df)
+
+    figuresize = 10
+    timelapse = settings['timelapse']
+    batch_size = settings['batch_size']
+
+    # Get object settings and initialize MEDIARPredictor
+    mediar_predictor = MEDIARPredictor(input_path=None, output_path=None, normalize=settings['normalize'], use_tta=False)
+
+    # Paths to input npz files
+    paths = [os.path.join(src, file) for file in os.listdir(src) if file.endswith('.npz')]
+
+    # Initialize a database for saving measurements
+    count_loc = os.path.join(os.path.dirname(src), 'measurements', 'measurements.db')
+    os.makedirs(os.path.dirname(src) + '/measurements', exist_ok=True)
+    _create_database(count_loc)
+
+    for file_index, path in enumerate(paths):
+        name = os.path.basename(path)
+        name, ext = os.path.splitext(name)
+        output_folder = os.path.join(os.path.dirname(path), f'{object_type}_mask_stack')
+        os.makedirs(output_folder, exist_ok=True)
+
+        with np.load(path) as data:
+            stack = data['data']
+            filenames = data['filenames']
+
+            for i, filename in enumerate(filenames):
+                output_path = os.path.join(output_folder, filename)
+                if os.path.exists(output_path):
+                    print(f"File {filename} already exists. Skipping...")
+                    continue
+
+        # Process each batch of images in the stack
+        for i in range(0, stack.shape[0], batch_size):
+            batch = stack[i: i + batch_size]
+            batch_filenames = filenames[i: i + batch_size]
+
+            # Prepare batch for MEDIARPredictor (optional)
+            batch = prepare_batch_for_segmentation(batch)
+
+            # Predict masks using MEDIARPredictor
+            predicted_masks = mediar_predictor.predict_batch(batch)
+
+            # Save predicted masks
+            for j, mask in enumerate(predicted_masks):
+                output_filename = os.path.join(output_folder, batch_filenames[j])
+                mask = mask.astype(np.uint16)
+                np.save(output_filename, mask)
+
+            # Optional: Plot the masks
+            if settings['plot']:
+                for idx, mask in enumerate(predicted_masks):
+                    plot_masks(batch[idx], mask, cmap='inferno', figuresize=figuresize)
+
+            # Save object counts to database
+            _save_object_counts_to_database(predicted_masks, object_type, batch_filenames, count_loc)
+
+        # Clear CUDA cache after each file
+        gc.collect()
+        torch.cuda.empty_cache()
+
+    print("Mask generation completed.")

{spacr-0.2.81 → spacr-0.3.1}/spacr/gui.py

@@ -32,7 +32,7 @@ class MainApp(tk.Tk):
         # Set the window size to the dimensions of the monitor where it is located
         self.geometry(f"{width}x{height}")
         self.title("SpaCr GUI Collection")
-        self.configure(bg='#333333')  # Set window background to dark gray
+        self.configure(bg='#333333')
 
         style = ttk.Style()
         self.color_settings = set_dark_style(style, parent_frame=self)
@@ -55,7 +55,8 @@ class MainApp(tk.Tk):
             "Cellpose All": (lambda frame: initiate_root(self, 'cellpose_all'), "Run Cellpose on all images."),
             "Map Barcodes": (lambda frame: initiate_root(self, 'map_barcodes'), "Map barcodes to data."),
             "Regression": (lambda frame: initiate_root(self, 'regression'), "Perform regression analysis."),
-            "Recruitment": (lambda frame: initiate_root(self, 'recruitment'), "Analyze recruitment data.")
+            "Recruitment": (lambda frame: initiate_root(self, 'recruitment'), "Analyze recruitment data."),
+            "Plaque": (lambda frame: initiate_root(self, 'analyze_plaques'), "Analyze plaque data.")
         }
 
         self.selected_app = tk.StringVar()

{spacr-0.2.81 → spacr-0.3.1}/spacr/gui_core.py

@@ -440,6 +440,8 @@ def import_settings(settings_type='mask'):
         settings = set_default_umap_image_settings(settings={})
     elif settings_type == 'recruitment':
         settings = get_analyze_recruitment_default_settings(settings={})
+    elif settings_type == 'analyze_plaques':
+        settings = {}
     else:
         raise ValueError(f"Invalid settings type: {settings_type}")
     
@@ -493,6 +495,8 @@ def setup_settings_panel(vertical_container, settings_type='mask'):
         settings = get_perform_regression_default_settings(settings={})
     elif settings_type == 'recruitment':
         settings = get_analyze_recruitment_default_settings(settings={})
+    elif settings_type == 'analyze_plaques':
+        settings = {}
     else:
         raise ValueError(f"Invalid settings type: {settings_type}")
 
@@ -645,7 +649,7 @@ def setup_usage_panel(horizontal_container, btn_col, uppdate_frequency):
     widgets = [usage_scrollable_frame.scrollable_frame]
     
     usage_bars = []
-    max_elements_per_column = 6
+    max_elements_per_column = 5
     row = 0
     col = 0
 
@@ -773,9 +777,9 @@ def start_process(q=None, fig_queue=None, settings_type='mask'):
     initialize_cuda()
 
     process_args = (settings_type, settings, q, fig_queue, stop_requested)
-    if settings_type in ['mask', 'umap', 'measure', 'simulation', 'sequencing', 
-                         'classify', 'cellpose_dataset', 'train_cellpose', 'ml_analyze', 'cellpose_masks', 'cellpose_all', 'map_barcodes', 
-                         'regression', 'recruitment', 'plaques', 'cellpose_compare', 'vision_scores', 'vision_dataset']:
+    if settings_type in ['mask', 'umap', 'measure', 'simulation', 'sequencing', 'classify', 'analyze_plaques', 
+                         'cellpose_dataset', 'train_cellpose', 'ml_analyze', 'cellpose_masks', 'cellpose_all', 'map_barcodes', 
+                         'regression', 'recruitment', 'cellpose_compare', 'vision_scores', 'vision_dataset']:
 
         # Start the process
         process = Process(target=run_function_gui, args=process_args)

{spacr-0.2.81 → spacr-0.3.1}/spacr/gui_utils.py

@@ -486,7 +486,7 @@ def function_gui_wrapper(function=None, settings={}, q=None, fig_queue=None, imp
 def run_function_gui(settings_type, settings, q, fig_queue, stop_requested):
     
     from .gui_utils import process_stdout_stderr
-    from .core import generate_image_umap, preprocess_generate_masks, generate_ml_scores, identify_masks_finetune, check_cellpose_models, analyze_recruitment, train_cellpose, compare_cellpose_masks, analyze_plaques, generate_dataset, apply_model_to_tar
+    from .core import generate_image_umap, preprocess_generate_masks, generate_ml_scores, identify_masks_finetune, check_cellpose_models, analyze_recruitment, train_cellpose, analyze_plaques, compare_cellpose_masks, generate_dataset, apply_model_to_tar
     from .io import generate_cellpose_train_test
     from .measure import measure_crop
     from .sim import run_multiple_simulations
@@ -532,6 +532,9 @@ def run_function_gui(settings_type, settings, q, fig_queue, stop_requested):
     elif settings_type == 'umap':
         function = generate_image_umap
         imports = 1
+    elif settings_type == 'analyze_plaques':
+        function = analyze_plaques
+        imports = 1
     else:
         raise ValueError(f"Invalid settings type: {settings_type}")
     try:

{spacr-0.2.81 → spacr-0.3.1}/spacr/io.py

@@ -26,7 +26,7 @@ import atexit
 
 from .logger import log_function_call
 
-def _load_images_and_labels(image_files, label_files, circular=False, invert=False, image_extension="*.tif", label_extension="*.tif"):
+def _load_images_and_labels(image_files, label_files, circular=False, invert=False):
     
     from .utils import invert_image, apply_mask
     
@@ -653,7 +653,6 @@ def _rename_and_organize_image_files(src, regex, batch_size=100, pick_slice=Fals
     from .utils import _extract_filename_metadata, print_progress
     
     regular_expression = re.compile(regex)
-    images_by_key = defaultdict(list)
     stack_path = os.path.join(src, 'stack')
     files_processed = 0
     if not os.path.exists(stack_path) or (os.path.isdir(stack_path) and len(os.listdir(stack_path)) == 0):
@@ -665,7 +664,7 @@ def _rename_and_organize_image_files(src, regex, batch_size=100, pick_slice=Fals
             start = time.time()
             batch_filenames = all_filenames[idx:idx+batch_size]
             for filename in batch_filenames:
-                images_by_key = _extract_filename_metadata(batch_filenames, src, images_by_key, regular_expression, metadata_type, pick_slice, skip_mode)
+                images_by_key = _extract_filename_metadata(batch_filenames, src, regular_expression, metadata_type, pick_slice, skip_mode)
             
             if pick_slice:
                 for i, key in enumerate(images_by_key):
@@ -683,16 +682,16 @@ def _rename_and_organize_image_files(src, regex, batch_size=100, pick_slice=Fals
                     files_to_process = len(all_filenames)
                     print_progress(files_processed, files_to_process, n_jobs=1, time_ls=time_ls, batch_size=batch_size, operation_type='Preprocessing filenames')
 
-                    if os.path.exists(output_path):                        
-                        print(f'WARNING: A file with the same name already exists at location {output_filename}')
-                    else:
+                    #if os.path.exists(output_path):                        
+                    #    print(f'WARNING: A file with the same name already exists at location {output_filename}')
+                    if not os.path.exists(output_path):
                         mip_image.save(output_path)
             else:
                 for i, (key, images) in enumerate(images_by_key.items()):
-                    mip = np.max(np.stack(images), axis=0)
-                    mip_image = Image.fromarray(mip)
                     plate, well, field, channel = key[:4]
                     output_dir = os.path.join(src, channel)
+                    mip = np.max(np.stack(images), axis=0)
+                    mip_image = Image.fromarray(mip)
                     os.makedirs(output_dir, exist_ok=True)
                     output_filename = f'{plate}_{well}_{field}.tif'
                     output_path = os.path.join(output_dir, output_filename)
@@ -703,9 +702,9 @@ def _rename_and_organize_image_files(src, regex, batch_size=100, pick_slice=Fals
                     files_to_process = len(all_filenames)
                     print_progress(files_processed, files_to_process, n_jobs=1, time_ls=time_ls, batch_size=batch_size, operation_type='Preprocessing filenames')
 
-                    if os.path.exists(output_path):                        
-                        print(f'WARNING: A file with the same name already exists at location {output_filename}')
-                    else:
+                    #if os.path.exists(output_path):                        
+                    #    print(f'WARNING: A file with the same name already exists at location {output_filename}')
+                    if not os.path.exists(output_path):
                         mip_image.save(output_path)
             images_by_key.clear()
 
@@ -972,8 +971,8 @@ def _mip_all(src, include_first_chan=True):
         if filename.endswith('.npy'):
             # Load the array from the file.
             array = np.load(os.path.join(src, filename))
-            # Normalize the array using custom parameters (q1=2, q2=98).
-            array = normalize_to_dtype(array, q1=2, q2=98, percentiles=None)
+            # Normalize the array 
+            #array = normalize_to_dtype(array, q1=0, q2=99, percentiles=None)
 
             if array.ndim != 3: # Check if the array is not 3-dimensional.
                 # Log a message indicating a zero array will be generated due to unexpected dimensions.
@@ -1671,6 +1670,7 @@ def preprocess_img_data(settings):
                 if plot:
                     print(f'plotting {nr} images from {src}/stack')
                     plot_arrays(src+'/stack', figuresize, cmap, nr=nr, normalize=normalize)
+
                 if all_to_mip:
                     _mip_all(src+'/stack')
                     if plot:

{spacr-0.2.81 → spacr-0.3.1}/spacr/measure.py

@@ -998,10 +998,10 @@ def measure_crop(settings):
         src_fldr = os.path.join(src_fldr, 'merged')
         print(f"Changed source folder to: {src_fldr}")
 
-    if settings['save_measurements']:
-        source_folder = os.path.dirname(settings['src'])
-        os.makedirs(source_folder+'/measurements', exist_ok=True)
-        _create_database(source_folder+'/measurements/measurements.db')
+    #if settings['save_measurements']:
+        #source_folder = os.path.dirname(settings['src'])
+        #os.makedirs(source_folder+'/measurements', exist_ok=True)
+        #_create_database(source_folder+'/measurements/measurements.db')
     
     if settings['cell_mask_dim'] is None:
         settings['include_uninfected'] = True