spacr 0.4.15__py3-none-any.whl → 0.5.0__py3-none-any.whl

spacr/__init__.py

@@ -26,7 +26,7 @@ from . import submodules
 from . import openai
 from . import ml
 from . import toxo
-from . import cellpose
+from . import spacr_cellpose
 from . import sp_stats
 from . import logger
 
@@ -57,7 +57,7 @@ __all__ = [
     "openai",
     "ml",
     "toxo",
-    "cellpose",
+    "spacr_cellpose",
     "sp_stats",
     "logger"
 ]

spacr/core.py

@@ -9,12 +9,11 @@ warnings.filterwarnings("ignore", message="3D stack used, but stitch_threshold=0
 
 def preprocess_generate_masks(settings):
 
-    from .io import preprocess_img_data, _load_and_concatenate_arrays
+    from .io import preprocess_img_data, _load_and_concatenate_arrays, convert_to_yokogawa, convert_separate_files_to_yokogawa
     from .plot import plot_image_mask_overlay, plot_arrays
-    from .utils import _pivot_counts_table, check_mask_folder, adjust_cell_masks, print_progress, save_settings, delete_intermedeate_files, format_path_for_system, normalize_src_path
+    from .utils import _pivot_counts_table, check_mask_folder, adjust_cell_masks, print_progress, save_settings, delete_intermedeate_files, format_path_for_system, normalize_src_path, generate_image_path_map, copy_images_to_consolidated
     from .settings import set_default_settings_preprocess_generate_masks
-    
-    
+        
     if 'src' in settings:
         if not isinstance(settings['src'], (str, list)):
             ValueError(f'src must be a string or a list of strings')
@@ -25,19 +24,51 @@ def preprocess_generate_masks(settings):
     
     settings['src'] = normalize_src_path(settings['src'])
     
+    if settings['consolidate']:
+        image_map = generate_image_path_map(settings['src'])
+        copy_images_to_consolidated(image_map, settings['src'])
+        settings['src'] = os.path.join(settings['src'], 'consolidated')
+
     if isinstance(settings['src'], str):
         settings['src'] = [settings['src']]
 
     if isinstance(settings['src'], list):
         source_folders = settings['src']
         for source_folder in source_folders:
+            
             print(f'Processing folder: {source_folder}')
-            source_folder = format_path_for_system(source_folder)            
+            
+            source_folder = format_path_for_system(source_folder)   
             settings['src'] = source_folder
             src = source_folder
             settings = set_default_settings_preprocess_generate_masks(settings)
+            
+            if settings['metadata_type'] == 'auto':
+                if settings['custom_regex'] != None:
+                    try:
+                        print(f"using regex: {settings['custom_regex']}")
+                        convert_separate_files_to_yokogawa(folder=source_folder, regex=settings['custom_regex'])
+                    except:
+                        try:
+                            convert_to_yokogawa(folder=source_folder)
+                        except Exception as e:
+                            print(f"Error: Tried to convert image files and image file name metadata with regex {settings['custom_regex']} then without regex but failed both.")
+                            print(f'Error: {e}')
+                            return
+                else:
+                    try:
+                        convert_to_yokogawa(folder=source_folder)
+                    except Exception as e:
+                        print(f"Error: Tried to convert image files and image file name metadata without regex but failed.")
+                        print(f'Error: {e}')
+                        return
+            
+            if settings['cell_channel'] is None and settings['nucleus_channel'] is None and settings['pathogen_channel'] is None:
+                print(f'Error: At least one of cell_channel, nucleus_channel or pathogen_channel must be defined')
+                return
+            
             save_settings(settings, name='gen_mask_settings')
-
+            
             if not settings['pathogen_channel'] is None:
                 custom_model_ls = ['toxo_pv_lumen','toxo_cyto']
                 if settings['pathogen_model'] not in custom_model_ls:
@@ -57,7 +88,7 @@ def preprocess_generate_masks(settings):
                 settings_df = pd.DataFrame(list(settings.items()), columns=['setting_key', 'setting_value'])
                 settings_df['setting_value'] = settings_df['setting_value'].apply(str)
                 display(settings_df)
-
+                
             if settings['test_mode']:
                 print(f'Starting Test mode ...')
 
@@ -136,7 +167,6 @@ def preprocess_generate_masks(settings):
                 
                 if settings['plot']:
                     if not settings['timelapse']:
-
                         if settings['test_mode'] == True:
                             settings['examples_to_plot'] = len(os.path.join(src,'merged'))
 
@@ -167,6 +197,7 @@ def preprocess_generate_masks(settings):
             gc.collect()
             
             if settings['delete_intermediate']:
+                print(f"deleting intermediate files")
                 delete_intermedeate_files(settings)
 
             print("Successfully completed run")
@@ -288,6 +319,17 @@ def generate_cellpose_masks(src, settings, object_type):
                 continue
             
             batch = prepare_batch_for_segmentation(batch)
+            
+            
+            #if settings['denoise']:
+            #    if object_type == 'cell':
+            #        model_type = "denoise_cyto3"
+            #    elif object_type == 'nucleus':
+            #        model_type = "denoise_nucleus"
+            #    else:
+            #        raise ValueError(f"No denoise model for object_type: {object_type}")
+            #    dn = denoise.DenoiseModel(model_type=model_type, gpu=device)
+            #    batch = dn.eval(imgs=batch, channels=chans, diameter=object_settings['diameter'])
 
             if timelapse:
                 movie_path = os.path.join(os.path.dirname(src), 'movies')
@@ -489,7 +531,7 @@ def generate_image_umap(settings={}):
         all_df = pd.concat([all_df, df], axis=0)
         #image_paths.extend(image_paths_tmp)
         
-    all_df['cond'] = all_df['column_name'].apply(map_condition, neg=settings['neg'], pos=settings['pos'], mix=settings['mix'])
+    all_df['cond'] = all_df['columnID'].apply(map_condition, neg=settings['neg'], pos=settings['pos'], mix=settings['mix'])
 
     if settings['exclude_conditions']:
         if isinstance(settings['exclude_conditions'], str):
@@ -673,7 +715,7 @@ def reducer_hyperparameter_search(settings={}, reduction_params=None, dbscan_par
         df = _read_and_join_tables(db_path, table_names=tables)
         all_df = pd.concat([all_df, df], axis=0)
 
-    all_df['cond'] = all_df['column_name'].apply(map_condition, neg=settings['neg'], pos=settings['pos'], mix=settings['mix'])
+    all_df['cond'] = all_df['columnID'].apply(map_condition, neg=settings['neg'], pos=settings['pos'], mix=settings['mix'])
 
     if settings['exclude_conditions']:
         if isinstance(settings['exclude_conditions'], str):

spacr/deep_spacr.py

@@ -1144,7 +1144,7 @@ def model_fusion(model_paths,save_path,device='cpu',model_name='maxvit_t',pretra
 
 def annotate_filter_vision(settings):
     
-    from .utils import annotate_conditions
+    from .utils import annotate_conditions, correct_metadata
     
     def filter_csv_by_png(csv_file):
         """
@@ -1188,8 +1188,7 @@ def annotate_filter_vision(settings):
 
         df = pd.read_csv(src)
         
-        if 'column_name' not in df.columns:
-            df['column_name'] = df['column']
+        df = correct_metadata(df)
             
         df = annotate_conditions(df, 
                             cells=settings['cells'],

spacr/gui.py

@@ -48,7 +48,6 @@ class MainApp(tk.Tk):
         }
 
         self.additional_gui_apps = {
-            "Convert": (lambda frame: initiate_root(self, 'convert'), "Convert images to Grayscale TIFs."),
             "Umap": (lambda frame: initiate_root(self, 'umap'), "Generate UMAP embeddings with datapoints represented as images."),
             "Train Cellpose": (lambda frame: initiate_root(self, 'train_cellpose'), "Train custom Cellpose models."),
             "ML Analyze": (lambda frame: initiate_root(self, 'ml_analyze'), "Machine learning analysis of data."),

spacr/gui_core.py

@@ -176,36 +176,6 @@ def display_figure(fig):
         elif event.num == 5: # Scroll down
             print("zoom out")
         
-    def zoom_v1(event):
-        # Fixed zoom factors (adjust these if you want faster or slower zoom)
-        zoom_in_factor = 0.9   # When zooming in, ranges shrink by 10%
-        zoom_out_factor = 1.1  # When zooming out, ranges increase by 10%
-
-        # Determine the zoom direction based on the scroll event
-        if event.num == 4 or (hasattr(event, 'delta') and event.delta > 0):  # Scroll up = zoom in
-            factor = zoom_in_factor
-        elif event.num == 5 or (hasattr(event, 'delta') and event.delta < 0): # Scroll down = zoom out
-            factor = zoom_out_factor
-        else:
-            return  # No recognized scroll direction
-
-        for ax in canvas.figure.get_axes():
-            xlim = ax.get_xlim()
-            ylim = ax.get_ylim()
-
-            x_center = (xlim[1] + xlim[0]) / 2
-            y_center = (ylim[1] + ylim[0]) / 2
-
-            x_range = (xlim[1] - xlim[0]) * factor
-            y_range = (ylim[1] - ylim[0]) * factor
-
-            # Set the new limits
-            ax.set_xlim([x_center - x_range / 2, x_center + x_range / 2])
-            ax.set_ylim([y_center - y_range / 2, y_center + y_range / 2])
-
-        # Redraw the figure efficiently
-        canvas.draw_idle()
-        
     def zoom(event):
         # Fixed zoom factors (adjust these if you want faster or slower zoom)
         zoom_in_factor = 0.9   # When zooming in, ranges shrink by 10%
@@ -832,7 +802,7 @@ def initiate_abort():
     global thread_control, q, parent_frame
     if thread_control.get("run_thread") is not None:
         try:
-            q.put("Aborting processes...")
+            #q.put("Aborting processes...")
             thread_control.get("run_thread").terminate()
             thread_control["run_thread"] = None
             q.put("Processes aborted.")
@@ -840,22 +810,164 @@ def initiate_abort():
             q.put(f"Error aborting process: {e}")
 
     thread_control = {"run_thread": None, "stop_requested": False}
+    
+def check_src_folders_files(settings, settings_type, q):
+    """
+    Checks if 'src' is a key in the settings dictionary and if it exists as a valid path.
+    If 'src' is a list, iterates through the list and checks each path.
+    If any path is missing, prompts the user to edit or remove invalid paths.
+    """
+
+    request_stop = False
+        
+    def _folder_has_images(folder_path, image_extensions = {".png", ".jpg", ".jpeg", ".bmp", ".gif", ".tiff", ".tif", ".webp", ".npy", ".npz", "nd2", "czi", "lif"}):
+        """Check if a folder contains any image files."""
+        return any(file.lower().endswith(tuple(image_extensions)) for file in os.listdir(folder_path))
+
+    def _has_folder(parent_folder, sub_folder="measure"):
+        """Check if a specific sub-folder exists inside the given folder."""
+        return os.path.isdir(os.path.join(parent_folder, sub_folder))
+    
+    from .utils import normalize_src_path, generate_image_path_map
+    
+    settings['src'] = normalize_src_path(settings['src'])
+
+    src_value = settings.get("src")
+
+    # **Skip if 'src' is missing**
+    if src_value is None:
+        return request_stop
+
+    # Convert single string src to a list for uniform handling
+    if isinstance(src_value, str):
+        src_list = [src_value]
+    elif isinstance(src_value, list):
+        src_list = src_value
+    else:
+        request_stop = True
+        return request_stop  # Ensure early exit
+
+    # Identify missing paths
+    missing_paths = {i: path for i, path in enumerate(src_list) if not os.path.exists(path)}
+
+    if missing_paths:
+        q.put(f'Error: The following paths are missing: {missing_paths}')
+        request_stop = True
+        return request_stop  # Ensure early exit
+
+    conditions = [True]  # Initialize conditions list
+
+    for path in src_list:  # Fixed: Use src_list instead of src_value
+        if settings_type == 'mask':
+            if settings['consolidate']:
+                image_map = generate_image_path_map(path)
+                if len(image_map) > 0:
+                    request_stop = False
+                    return request_stop
+                else:
+                    q.put(f"Error: Missing subfolders with images for: {path}")
+                    request_stop = True
+                    return request_stop
+            else:
+                pictures_continue = _folder_has_images(path)
+                folder_chan_continue = _has_folder(path, "1")
+                folder_stack_continue = _has_folder(path, "stack")
+                folder_npz_continue = _has_folder(path, "norm_channel_stack")
+            
+                if not pictures_continue:
+                    if not any([folder_chan_continue, folder_stack_continue, folder_npz_continue]):
+                        if not folder_chan_continue:
+                            q.put(f"Error: Missing channel folder in folder: {path}")
+                            
+                        if not folder_stack_continue:
+                            q.put(f"Error: Missing stack folder in folder: {path}")
+                            
+                        if not folder_npz_continue:
+                            q.put(f"Error: Missing norm_channel_stack folder in folder: {path}")
+                        else:
+                            q.put(f"Error: No images in folder: {path}")
+                
+                #q.put(f"path:{path}")
+                #q.put(f"pictures_continue:{pictures_continue}, folder_chan_continue:{folder_chan_continue}, folder_stack_continue:{folder_stack_continue}, folder_npz_continue:{folder_npz_continue}")
+
+                conditions = [pictures_continue, folder_chan_continue, folder_stack_continue, folder_npz_continue]
+            
+        if settings_type == 'measure':
+            if not os.path.basename(path) == 'merged':
+                path = os.path.join(path, "merged")
+            npy_continue = _folder_has_images(path, image_extensions={".npy"})
+            conditions = [npy_continue]
+            
+        #if settings_type == 'recruitment':
+        #    if not os.path.basename(path) == 'measurements':
+        #        path = os.path.join(path, "measurements")
+        #    db_continue = _folder_has_images(path, image_extensions={".db"})
+        #    conditions = [db_continue]
+            
+        #if settings_type == 'umap':
+        #    if not os.path.basename(path) == 'measurements':
+        #        path = os.path.join(path, "measurements")
+        #    db_continue = _folder_has_images(path, image_extensions={".db"})
+        #    conditions = [db_continue]
+            
+        #if settings_type == 'analyze_plaques':
+        #    if not os.path.basename(path) == 'measurements':
+        #        path = os.path.join(path, "measurements")
+        #    db_continue = _folder_has_images(path, image_extensions={".db"})
+        #    conditions = [db_continue]
+        
+        #if settings_type == 'map_barcodes':
+        #    if not os.path.basename(path) == 'measurements':
+        #        path = os.path.join(path, "measurements")
+        #    db_continue = _folder_has_images(path, image_extensions={".db"})
+        #    conditions = [db_continue]
+        
+        #if settings_type == 'regression':
+        #    if not os.path.basename(path) == 'measurements':
+        #        path = os.path.join(path, "measurements")
+        #    db_continue = _folder_has_images(path, image_extensions={".db"})
+        #    conditions = [db_continue]
+            
+        #if settings_type == 'classify':
+        #    if not os.path.basename(path) == 'measurements':
+        #        path = os.path.join(path, "measurements")
+        #    db_continue = _folder_has_images(path, image_extensions={".db"})
+        #    conditions = [db_continue]
+            
+        #if settings_type == 'analyze_plaques':
+        #    if not os.path.basename(path) == 'measurements':
+        #        path = os.path.join(path, "measurements")
+        #    db_continue = _folder_has_images(path, image_extensions={".db"})
+        #    conditions = [db_continue]
+            
+    if not any(conditions):
+        q.put(f"Error: The following path(s) is missing images or folders: {path}")
+        request_stop = True
+            
+    return request_stop
 
 def start_process(q=None, fig_queue=None, settings_type='mask'):
     global thread_control, vars_dict, parent_frame
     from .settings import check_settings, expected_types
     from .gui_utils import run_function_gui, set_cpu_affinity, initialize_cuda, display_gif_in_plot_frame, print_widget_structure
-
+        
     if q is None:
         q = Queue()
     if fig_queue is None:
         fig_queue = Queue()
     try:
-        settings = check_settings(vars_dict, expected_types, q)
+        settings, errors = check_settings(vars_dict, expected_types, q)
+        
+        if len(errors) > 0:
+            return
+        
+        if check_src_folders_files(settings, settings_type, q):
+            return
+    
     except ValueError as e:
         q.put(f"Error: {e}")
         return
-
+        
     if isinstance(thread_control, dict) and thread_control.get("run_thread") is not None:
         initiate_abort()
     
@@ -880,10 +992,11 @@ def start_process(q=None, fig_queue=None, settings_type='mask'):
 
         # Store the process in thread_control for future reference
         thread_control["run_thread"] = process
+        
     else:
         q.put(f"Error: Unknown settings type '{settings_type}'")
         return
-
+    
 def process_console_queue():
     global q, console_output, parent_frame, progress_bar, process_console_queue
 
@@ -894,13 +1007,105 @@ def process_console_queue():
         process_console_queue.current_maximum = None
 
     ansi_escape_pattern = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
+    
+    spacing = 5
+
+    # **Configure styles for different message types**
+    console_output.tag_configure("error", foreground="red", spacing3 = spacing)
+    console_output.tag_configure("warning", foreground="orange", spacing3 = spacing)
+    console_output.tag_configure("normal", foreground="white", spacing3 = spacing)
 
     while not q.empty():
         message = q.get_nowait()
         clean_message = ansi_escape_pattern.sub('', message)
-        #console_output.insert(tk.END, clean_message + "\n")
-        #console_output.see(tk.END)
+
+        # **Detect Error Messages (Red)**
+        if clean_message.startswith("Error:"):
+            console_output.insert(tk.END, clean_message + "\n", "error")
+            console_output.see(tk.END)
+            #print("Run aborted due to error:", clean_message)  # Debug message
+            #return  # **Exit immediately to stop further execution**
+
+        # **Detect Warning Messages (Orange)**
+        elif clean_message.startswith("Warning:"):
+            console_output.insert(tk.END, clean_message + "\n", "warning")
         
+        # **Process Progress Messages Normally**
+        elif clean_message.startswith("Progress:"):
+            try:
+                # Extract the progress information
+                match = re.search(r'Progress: (\d+)/(\d+), operation_type: ([\w\s]*),(.*)', clean_message)
+
+                if match:
+                    current_progress = int(match.group(1))
+                    total_progress = int(match.group(2))
+                    operation_type = match.group(3).strip()
+                    additional_info = match.group(4).strip()  # Capture everything after operation_type
+                    
+                    # Check if the maximum value has changed
+                    if process_console_queue.current_maximum != total_progress:
+                        process_console_queue.current_maximum = total_progress
+                        process_console_queue.completed_tasks = []
+
+                    # Add the task to the completed set
+                    process_console_queue.completed_tasks.append(current_progress)
+                    
+                    # Calculate the unique progress count
+                    unique_progress_count = len(np.unique(process_console_queue.completed_tasks))
+
+                    # Update the progress bar
+                    if progress_bar:
+                        progress_bar['maximum'] = total_progress
+                        progress_bar['value'] = unique_progress_count
+
+                    # Store operation type and additional info
+                    if operation_type:
+                        progress_bar.operation_type = operation_type
+                        progress_bar.additional_info = additional_info
+
+                    # Update the progress label
+                    if progress_bar.progress_label:
+                        progress_bar.update_label()
+
+                    # Clear completed tasks when progress is complete
+                    if unique_progress_count >= total_progress:
+                        process_console_queue.completed_tasks.clear()
+
+            except Exception as e:
+                print(f"Error parsing progress message: {e}")
+
+        # **Insert Normal Messages with Extra Line Spacing**
+        else:
+            console_output.insert(tk.END, clean_message + "\n", "normal")
+
+        console_output.see(tk.END)
+
+    # **Continue processing if no error was detected**
+    after_id = console_output.after(uppdate_frequency, process_console_queue)
+    parent_frame.after_tasks.append(after_id)
+    
+def process_console_queue_v2():
+    global q, console_output, parent_frame, progress_bar, process_console_queue
+
+    # Initialize function attribute if it doesn't exist
+    if not hasattr(process_console_queue, "completed_tasks"):
+        process_console_queue.completed_tasks = []
+    if not hasattr(process_console_queue, "current_maximum"):
+        process_console_queue.current_maximum = None
+
+    ansi_escape_pattern = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
+
+    while not q.empty():
+        message = q.get_nowait()
+        clean_message = ansi_escape_pattern.sub('', message)
+
+        # **Abort Execution if an Error Message is Detected**
+        if clean_message.startswith("Error:"):
+            console_output.insert(tk.END, clean_message + "\n", "error")
+            console_output.see(tk.END)
+            print("Run aborted due to error:", clean_message)  # Debug message
+            return  # **Exit immediately to stop further execution**
+
         # Check if the message contains progress information
         if clean_message.startswith("Progress:"):
             try:
@@ -928,8 +1133,7 @@ def process_console_queue():
                     if progress_bar:
                         progress_bar['maximum'] = total_progress
                         progress_bar['value'] = unique_progress_count
-                        #print("Current progress bar value:", progress_bar['value']) # Debugg
-                        
+
                     # Store operation type and additional info
                     if operation_type:
                         progress_bar.operation_type = operation_type
@@ -945,11 +1149,13 @@ def process_console_queue():
 
             except Exception as e:
                 print(f"Error parsing progress message: {e}")
+
         else:
-            # Only insert messages that do not start with "Progress:"
+            # Insert non-progress messages into the console
             console_output.insert(tk.END, clean_message + "\n")
             console_output.see(tk.END)
-        
+
+    # **Continue processing if no error was detected**
     after_id = console_output.after(uppdate_frequency, process_console_queue)
     parent_frame.after_tasks.append(after_id)