spacr 1.0.6__py3-none-any.whl → 1.0.7__py3-none-any.whl

spacr/__init__.py

@@ -15,21 +15,17 @@ from . import utils
 from . import settings
 from . import plot
 from . import measure
-from . import sim
 from . import sequencing
 from . import timelapse
 from . import deep_spacr
-from . import app_annotate
 from . import gui_utils
 from . import gui_elements
 from . import gui_core
 from . import gui
-from . import app_make_masks
 from . import app_mask
 from . import app_measure
 from . import app_classify
 from . import app_sequencing
-from . import app_umap
 from . import submodules
 from . import ml
 from . import toxo
@@ -44,23 +40,18 @@ __all__ = [
     "settings",
     "plot",
     "measure",
-    "sim",
     "sequencing",
     "timelapse",
     "deep_spacr",
-    "app_annotate",
     "gui_utils",
     "gui_elements",
     "gui_core",
     "gui",
-    "app_make_masks",
     "app_mask",
     "app_measure",
     "app_classify",
     "app_sequencing",
-    "app_umap",
     "submodules",
-    "openai",
     "ml",
     "toxo",
     "spacr_cellpose",

spacr/core.py

@@ -1,7 +1,6 @@
 import os, gc, torch, time, random
 import numpy as np
 import pandas as pd
-import matplotlib.pyplot as plt
 from IPython.display import display
 
 import warnings
@@ -449,362 +448,6 @@ def generate_cellpose_masks(src, settings, object_type):
     torch.cuda.empty_cache()
     return
 
-def generate_image_umap(settings={}):
-    """
-    Generate UMAP or tSNE embedding and visualize the data with clustering.
-    
-    Parameters:
-    settings (dict): Dictionary containing the following keys:
-    src (str): Source directory containing the data.
-    row_limit (int): Limit the number of rows to process.
-    tables (list): List of table names to read from the database.
-    visualize (str): Visualization type.
-    image_nr (int): Number of images to display.
-    dot_size (int): Size of dots in the scatter plot.
-    n_neighbors (int): Number of neighbors for UMAP.
-    figuresize (int): Size of the figure.
-    black_background (bool): Whether to use a black background.
-    remove_image_canvas (bool): Whether to remove the image canvas.
-    plot_outlines (bool): Whether to plot outlines.
-    plot_points (bool): Whether to plot points.
-    smooth_lines (bool): Whether to smooth lines.
-    verbose (bool): Whether to print verbose output.
-    embedding_by_controls (bool): Whether to use embedding from controls.
-    col_to_compare (str): Column to compare for control-based embedding.
-    pos (str): Positive control value.
-    neg (str): Negative control value.
-    clustering (str): Clustering method ('DBSCAN' or 'KMeans').
-    exclude (list): List of columns to exclude from the analysis.
-    plot_images (bool): Whether to plot images.
-    reduction_method (str): Dimensionality reduction method ('UMAP' or 'tSNE').
-    save_figure (bool): Whether to save the figure as a PDF.
-    
-    Returns:
-    pd.DataFrame: DataFrame with the original data and an additional column 'cluster' containing the cluster identity.
-    """
- 
-    from .io import _read_and_join_tables
-    from .utils import get_db_paths, preprocess_data, reduction_and_clustering, remove_noise, generate_colors, correct_paths, plot_embedding, plot_clusters_grid, cluster_feature_analysis, map_condition
-    from .settings import set_default_umap_image_settings
-    settings = set_default_umap_image_settings(settings)
-
-    if isinstance(settings['src'], str):
-        settings['src'] = [settings['src']]
-
-    if settings['plot_images'] is False:
-        settings['black_background'] = False
-
-    if settings['color_by']:
-        settings['remove_cluster_noise'] = False
-        settings['plot_outlines'] = False
-        settings['smooth_lines'] = False
-
-    print(f'Generating Image UMAP ...')
-    settings_df = pd.DataFrame(list(settings.items()), columns=['Key', 'Value'])
-    settings_dir = os.path.join(settings['src'][0],'settings')
-    settings_csv = os.path.join(settings_dir,'embedding_settings.csv')
-    os.makedirs(settings_dir, exist_ok=True)
-    settings_df.to_csv(settings_csv, index=False)
-    display(settings_df)
-
-    db_paths = get_db_paths(settings['src'])
-    tables = settings['tables'] + ['png_list']
-    all_df = pd.DataFrame()
-
-    for i,db_path in enumerate(db_paths):
-        df = _read_and_join_tables(db_path, table_names=tables)
-        df, image_paths_tmp = correct_paths(df, settings['src'][i])
-        all_df = pd.concat([all_df, df], axis=0)
-        #image_paths.extend(image_paths_tmp)
-        
-    all_df['cond'] = all_df['columnID'].apply(map_condition, neg=settings['neg'], pos=settings['pos'], mix=settings['mix'])
-
-    if settings['exclude_conditions']:
-        if isinstance(settings['exclude_conditions'], str):
-            settings['exclude_conditions'] = [settings['exclude_conditions']]
-        row_count_before = len(all_df)
-        all_df = all_df[~all_df['cond'].isin(settings['exclude_conditions'])]
-        if settings['verbose']:
-            print(f'Excluded {row_count_before - len(all_df)} rows after excluding: {settings["exclude_conditions"]}, rows left: {len(all_df)}')
-
-    if settings['row_limit'] is not None:
-        all_df = all_df.sample(n=settings['row_limit'], random_state=42)
-
-    image_paths = all_df['png_path'].to_list()
-
-    if settings['embedding_by_controls']:
-        
-        # Extract and reset the index for the column to compare
-        col_to_compare = all_df[settings['col_to_compare']].reset_index(drop=True)
-        print(col_to_compare)
-        #if settings['only_top_features']:
-        #    column_list = None
-            
-        # Preprocess the data to obtain numeric data
-        numeric_data = preprocess_data(all_df, settings['filter_by'], settings['remove_highly_correlated'], settings['log_data'], settings['exclude'])
-
-        # Convert numeric_data back to a DataFrame to align with col_to_compare
-        numeric_data_df = pd.DataFrame(numeric_data)
-
-        # Ensure numeric_data_df and col_to_compare are properly aligned
-        numeric_data_df = numeric_data_df.reset_index(drop=True)
-
-        # Assign the column back to numeric_data_df
-        numeric_data_df[settings['col_to_compare']] = col_to_compare
-
-        # Subset the dataframe based on specified column values for controls
-        positive_control_df = numeric_data_df[numeric_data_df[settings['col_to_compare']] == settings['pos']].copy()
-        negative_control_df = numeric_data_df[numeric_data_df[settings['col_to_compare']] == settings['neg']].copy()
-        control_numeric_data_df = pd.concat([positive_control_df, negative_control_df])
-
-        # Drop the comparison column from numeric_data_df and control_numeric_data_df
-        numeric_data_df = numeric_data_df.drop(columns=[settings['col_to_compare']])
-        control_numeric_data_df = control_numeric_data_df.drop(columns=[settings['col_to_compare']])
-
-        # Convert numeric_data_df and control_numeric_data_df back to numpy arrays
-        numeric_data = numeric_data_df.values
-        control_numeric_data = control_numeric_data_df.values
-
-        # Train the reducer on control data
-        _, _, reducer = reduction_and_clustering(control_numeric_data, settings['n_neighbors'], settings['min_dist'], settings['metric'], settings['eps'], settings['min_samples'], settings['clustering'], settings['reduction_method'], settings['verbose'], n_jobs=settings['n_jobs'], mode='fit', model=False)
-        
-        # Apply the trained reducer to the entire dataset
-        numeric_data = preprocess_data(all_df, settings['filter_by'], settings['remove_highly_correlated'], settings['log_data'], settings['exclude'])
-        embedding, labels, _ = reduction_and_clustering(numeric_data, settings['n_neighbors'], settings['min_dist'], settings['metric'], settings['eps'], settings['min_samples'], settings['clustering'], settings['reduction_method'], settings['verbose'], n_jobs=settings['n_jobs'], mode=None, model=reducer)
-
-    else:
-        if settings['resnet_features']:
-            # placeholder for resnet features, not implemented yet
-            pass
-            #numeric_data, embedding, labels = generate_umap_from_images(image_paths, settings['n_neighbors'], settings['min_dist'], settings['metric'], settings['clustering'], settings['eps'], settings['min_samples'], settings['n_jobs'], settings['verbose'])
-        else:
-            # Apply the trained reducer to the entire dataset
-            numeric_data = preprocess_data(all_df, settings['filter_by'], settings['remove_highly_correlated'], settings['log_data'], settings['exclude'])
-            embedding, labels, _ = reduction_and_clustering(numeric_data, settings['n_neighbors'], settings['min_dist'], settings['metric'], settings['eps'], settings['min_samples'], settings['clustering'], settings['reduction_method'], settings['verbose'], n_jobs=settings['n_jobs'])
-    
-    if settings['remove_cluster_noise']:
-        # Remove noise from the clusters (removes -1 labels from DBSCAN)
-        embedding, labels = remove_noise(embedding, labels)
-
-    # Plot the results
-    if settings['color_by']:
-        if settings['embedding_by_controls']:
-            labels = all_df[settings['color_by']]
-        else:
-            labels = all_df[settings['color_by']]
-    
-    # Generate colors for the clusters
-    colors = generate_colors(len(np.unique(labels)), settings['black_background'])
-
-    # Plot the embedding
-    umap_plt = plot_embedding(embedding, image_paths, labels, settings['image_nr'], settings['img_zoom'], colors, settings['plot_by_cluster'], settings['plot_outlines'], settings['plot_points'], settings['plot_images'], settings['smooth_lines'], settings['black_background'], settings['figuresize'], settings['dot_size'], settings['remove_image_canvas'], settings['verbose'])
-    if settings['plot_cluster_grids'] and settings['plot_images']:
-        grid_plt = plot_clusters_grid(embedding, labels, settings['image_nr'], image_paths, colors, settings['figuresize'], settings['black_background'], settings['verbose'])
-    
-    # Save figure as PDF if required
-    if settings['save_figure']:
-        results_dir = os.path.join(settings['src'][0], 'results')
-        os.makedirs(results_dir, exist_ok=True)
-        reduction_method = settings['reduction_method'].upper()
-        embedding_path = os.path.join(results_dir, f'{reduction_method}_embedding.pdf')
-        umap_plt.savefig(embedding_path, format='pdf')
-        print(f'Saved {reduction_method} embedding to {embedding_path} and grid to {embedding_path}')
-        if settings['plot_cluster_grids'] and settings['plot_images']:
-            grid_path = os.path.join(results_dir, f'{reduction_method}_grid.pdf')
-            grid_plt.savefig(grid_path, format='pdf')
-            print(f'Saved {reduction_method} embedding to {embedding_path} and grid to {grid_path}')
-
-    # Add cluster labels to the dataframe
-    if len(labels) > 0:
-        all_df['cluster'] = labels
-    else:
-        all_df['cluster'] = 1  # Assign a default cluster label
-        print("No clusters found. Consider reducing 'min_samples' or increasing 'eps' for DBSCAN.")
-
-    # Save the results to a CSV file
-    results_dir = os.path.join(settings['src'][0], 'results')
-    results_csv = os.path.join(results_dir,'embedding_results.csv')
-    os.makedirs(results_dir, exist_ok=True)
-    all_df.to_csv(results_csv, index=False)
-    print(f'Results saved to {results_csv}')
-
-    if settings['analyze_clusters']:
-        combined_results = cluster_feature_analysis(all_df)
-        results_dir = os.path.join(settings['src'][0], 'results')
-        cluster_results_csv = os.path.join(results_dir,'cluster_results.csv')
-        os.makedirs(results_dir, exist_ok=True)
-        combined_results.to_csv(cluster_results_csv, index=False)
-        print(f'Cluster results saved to {cluster_results_csv}')
-
-    return all_df
-
-def reducer_hyperparameter_search(settings={}, reduction_params=None, dbscan_params=None, kmeans_params=None, save=False):
-    """
-    Perform a hyperparameter search for UMAP or tSNE on the given data.
-    
-    Parameters:
-    settings (dict): Dictionary containing the following keys:
-    src (str): Source directory containing the data.
-    row_limit (int): Limit the number of rows to process.
-    tables (list): List of table names to read from the database.
-    filter_by (str): Column to filter the data.
-    sample_size (int): Number of samples to use for the hyperparameter search.
-    remove_highly_correlated (bool): Whether to remove highly correlated columns.
-    log_data (bool): Whether to log transform the data.
-    verbose (bool): Whether to print verbose output.
-    reduction_method (str): Dimensionality reduction method ('UMAP' or 'tSNE').
-    reduction_params (list): List of dictionaries containing hyperparameters to test for the reduction method.
-    dbscan_params (list): List of dictionaries containing DBSCAN hyperparameters to test.
-    kmeans_params (list): List of dictionaries containing KMeans hyperparameters to test.
-    pointsize (int): Size of the points in the scatter plot.
-    save (bool): Whether to save the resulting plot as a file.
-    
-    Returns:
-    None
-    """
-    
-    from .io import _read_and_join_tables
-    from .utils import get_db_paths, preprocess_data, search_reduction_and_clustering, generate_colors, map_condition
-    from .settings import set_default_umap_image_settings
-
-    settings = set_default_umap_image_settings(settings)
-    pointsize = settings['dot_size']
-    if isinstance(dbscan_params, dict):
-        dbscan_params = [dbscan_params]
-
-    if isinstance(kmeans_params, dict):
-        kmeans_params = [kmeans_params]
-
-    if isinstance(reduction_params, dict):
-        reduction_params = [reduction_params]
-
-    # Determine reduction method based on the keys in reduction_param
-    if any('n_neighbors' in param for param in reduction_params):
-        reduction_method = 'umap'
-    elif any('perplexity' in param for param in reduction_params):
-        reduction_method = 'tsne'
-    elif any('perplexity' in param for param in reduction_params) and any('n_neighbors' in param for param in reduction_params):
-        raise ValueError("Reduction parameters must include 'n_neighbors' for UMAP or 'perplexity' for tSNE, not both.")
-    
-    if settings['reduction_method'].lower() != reduction_method:
-        settings['reduction_method'] = reduction_method
-        print(f'Changed reduction method to {reduction_method} based on the provided parameters.')
-    
-    if settings['verbose']:
-        display(pd.DataFrame(list(settings.items()), columns=['Key', 'Value']))
-
-    db_paths = get_db_paths(settings['src'])
-    
-    tables = settings['tables']
-    all_df = pd.DataFrame()
-    for db_path in db_paths:
-        df = _read_and_join_tables(db_path, table_names=tables)
-        all_df = pd.concat([all_df, df], axis=0)
-
-    all_df['cond'] = all_df['columnID'].apply(map_condition, neg=settings['neg'], pos=settings['pos'], mix=settings['mix'])
-
-    if settings['exclude_conditions']:
-        if isinstance(settings['exclude_conditions'], str):
-            settings['exclude_conditions'] = [settings['exclude_conditions']]
-        row_count_before = len(all_df)
-        all_df = all_df[~all_df['cond'].isin(settings['exclude_conditions'])]
-        if settings['verbose']:
-            print(f'Excluded {row_count_before - len(all_df)} rows after excluding: {settings["exclude_conditions"]}, rows left: {len(all_df)}')
-
-    if settings['row_limit'] is not None:
-        all_df = all_df.sample(n=settings['row_limit'], random_state=42)
-
-    numeric_data = preprocess_data(all_df, settings['filter_by'], settings['remove_highly_correlated'], settings['log_data'], settings['exclude'])
-
-    # Combine DBSCAN and KMeans parameters
-    clustering_params = []
-    if dbscan_params:
-        for param in dbscan_params:
-            param['method'] = 'dbscan'
-            clustering_params.append(param)
-    if kmeans_params:
-        for param in kmeans_params:
-            param['method'] = 'kmeans'
-            clustering_params.append(param)
-
-    print('Testing paramiters:', reduction_params)
-    print('Testing clustering paramiters:', clustering_params)
-
-    # Calculate the grid size
-    grid_rows = len(reduction_params)
-    grid_cols = len(clustering_params)
-
-    fig_width = grid_cols*10
-    fig_height = grid_rows*10
-
-    fig, axs = plt.subplots(grid_rows, grid_cols, figsize=(fig_width, fig_height))
-
-    # Make sure axs is always an array of axes
-    axs = np.atleast_1d(axs)
-    
-    # Iterate through the Cartesian product of reduction and clustering hyperparameters
-    for i, reduction_param in enumerate(reduction_params):
-        for j, clustering_param in enumerate(clustering_params):
-            if len(clustering_params) <= 1:
-                axs[i].axis('off')
-                ax = axs[i]
-            elif len(reduction_params) <= 1:
-                axs[j].axis('off')
-                ax = axs[j]
-            else:
-                ax = axs[i, j]
-
-            # Perform dimensionality reduction and clustering
-            if settings['reduction_method'].lower() == 'umap':
-                n_neighbors = reduction_param.get('n_neighbors', 15)
-
-                if isinstance(n_neighbors, float):
-                    n_neighbors = int(n_neighbors * len(numeric_data))
-
-                min_dist = reduction_param.get('min_dist', 0.1)
-                embedding, labels = search_reduction_and_clustering(numeric_data, n_neighbors, min_dist, settings['metric'], 
-                                                                    clustering_param.get('eps', 0.5), clustering_param.get('min_samples', 5), 
-                                                                    clustering_param['method'], settings['reduction_method'], settings['verbose'], reduction_param, n_jobs=settings['n_jobs'])
-                
-            elif settings['reduction_method'].lower() == 'tsne':
-                perplexity = reduction_param.get('perplexity', 30)
-
-                if isinstance(perplexity, float):
-                    perplexity = int(perplexity * len(numeric_data))
-
-                embedding, labels = search_reduction_and_clustering(numeric_data, perplexity, 0.1, settings['metric'], 
-                                                                    clustering_param.get('eps', 0.5), clustering_param.get('min_samples', 5), 
-                                                                    clustering_param['method'], settings['reduction_method'], settings['verbose'], reduction_param, n_jobs=settings['n_jobs'])
-                
-            else:
-                raise ValueError(f"Unsupported reduction method: {settings['reduction_method']}. Supported methods are 'UMAP' and 'tSNE'")
-
-            # Plot the results
-            if settings['color_by']:
-                unique_groups = all_df[settings['color_by']].unique()
-                colors = generate_colors(len(unique_groups), False)
-                for group, color in zip(unique_groups, colors):
-                    indices = all_df[settings['color_by']] == group
-                    ax.scatter(embedding[indices, 0], embedding[indices, 1], s=pointsize, label=f"{group}", color=color)
-            else:
-                unique_labels = np.unique(labels)
-                colors = generate_colors(len(unique_labels), False)
-                for label, color in zip(unique_labels, colors):
-                    ax.scatter(embedding[labels == label, 0], embedding[labels == label, 1], s=pointsize, label=f"Cluster {label}", color=color)
-
-            ax.set_title(f"{settings['reduction_method']} {reduction_param}\n{clustering_param['method']} {clustering_param}")
-            ax.legend()
-
-    plt.tight_layout()
-    if save:
-        results_dir = os.path.join(settings['src'], 'results')
-        os.makedirs(results_dir, exist_ok=True)
-        plt.savefig(os.path.join(results_dir, 'hyperparameter_search.pdf'))
-    else:
-        plt.show()
-
-    return
-
 def generate_screen_graphs(settings):
     """
     Generate screen graphs for different measurements in a given source directory.

spacr/gui.py

@@ -44,22 +44,15 @@ class MainApp(tk.Tk):
         self.main_gui_apps = {
             "Mask": (lambda frame: initiate_root(self, 'mask'), "Generate cellpose masks for cells, nuclei and pathogen images."),
             "Measure": (lambda frame: initiate_root(self, 'measure'), "Measure single object intensity and morphological feature. Crop and save single object image"),
-            #"Annotate": (lambda frame: initiate_root(self, 'annotate'), "Annotation single object images on a grid. Annotations are saved to database."),
-            #"Make Masks": (lambda frame: initiate_root(self, 'make_masks'), "Adjust pre-existing Cellpose models to your specific dataset for improved performance"),
             "Classify": (lambda frame: initiate_root(self, 'classify'), "Train Torch Convolutional Neural Networks (CNNs) or Transformers to classify single object images."),
         }
 
         self.additional_gui_apps = {
-            #"Umap": (lambda frame: initiate_root(self, 'umap'), "Generate UMAP embeddings with datapoints represented as images."),
-            #"Train Cellpose": (lambda frame: initiate_root(self, 'train_cellpose'), "Train custom Cellpose models."),
-            #"ML Analyze": (lambda frame: initiate_root(self, 'ml_analyze'), "Machine learning analysis of data."),
-            #"Cellpose Masks": (lambda frame: initiate_root(self, 'cellpose_masks'), "Generate Cellpose masks."),
-            #"Cellpose All": (lambda frame: initiate_root(self, 'cellpose_all'), "Run Cellpose on all images."),
+            "ML Analyze": (lambda frame: initiate_root(self, 'ml_analyze'), "Machine learning analysis of data."),
             "Map Barcodes": (lambda frame: initiate_root(self, 'map_barcodes'), "Map barcodes to data."),
             "Regression": (lambda frame: initiate_root(self, 'regression'), "Perform regression analysis."),
-            #"Recruitment": (lambda frame: initiate_root(self, 'recruitment'), "Analyze recruitment data."),
+            "Recruitment": (lambda frame: initiate_root(self, 'recruitment'), "Analyze recruitment data."),
             "Activation": (lambda frame: initiate_root(self, 'activation'), "Generate activation maps of computer vision models and measure channel-activation correlation."),
-            #"Plaque": (lambda frame: initiate_root(self, 'analyze_plaques'), "Analyze plaque data.")
         }
 
         self.selected_app = tk.StringVar()

spacr/gui_core.py

@@ -1266,36 +1266,29 @@ def initiate_root(parent, settings_type='mask'):
     fig_queue = Queue()
     parent_frame, vertical_container, horizontal_container, settings_container = setup_frame(parent_frame)
 
-    if settings_type == 'annotate':
-        from .app_annotate import initiate_annotation_app
-        initiate_annotation_app(horizontal_container)
-    elif settings_type == 'make_masks':
-        from .app_make_masks import initiate_make_mask_app
-        initiate_make_mask_app(horizontal_container)
+    scrollable_frame, vars_dict = setup_settings_panel(settings_container, settings_type)
+    print('setup_settings_panel')
+    canvas, canvas_widget = setup_plot_section(vertical_container, settings_type)
+    console_output, _ = setup_console(vertical_container) #, chatbot)
+    button_scrollable_frame, btn_col = setup_button_section(horizontal_container, settings_type)
+    
+    if num_cores > 12:
+        _, usage_bars, btn_col = setup_usage_panel(horizontal_container, btn_col, uppdate_frequency)
     else:
-        scrollable_frame, vars_dict = setup_settings_panel(settings_container, settings_type)
-        print('setup_settings_panel')
-        canvas, canvas_widget = setup_plot_section(vertical_container, settings_type)
-        console_output, _ = setup_console(vertical_container) #, chatbot)
-        button_scrollable_frame, btn_col = setup_button_section(horizontal_container, settings_type)
-        
-        if num_cores > 12:
-            _, usage_bars, btn_col = setup_usage_panel(horizontal_container, btn_col, uppdate_frequency)
-        else:
-            usage_bars = []
+        usage_bars = []
 
-        set_globals(thread_control, q, console_output, parent_frame, vars_dict, canvas, canvas_widget, scrollable_frame, fig_queue, progress_bar, usage_bars)
-        description_text = descriptions.get(settings_type, "No description available for this module.")
-        
-        q.put(f"Console")
-        q.put(f" ")
-        q.put(description_text)
-        
-        process_console_queue()
-        process_fig_queue()
-        create_menu_bar(parent)
-        after_id = parent_window.after(uppdate_frequency, lambda: main_thread_update_function(parent_window, q, fig_queue, canvas_widget))
-        parent_window.after_tasks.append(after_id)
+    set_globals(thread_control, q, console_output, parent_frame, vars_dict, canvas, canvas_widget, scrollable_frame, fig_queue, progress_bar, usage_bars)
+    description_text = descriptions.get(settings_type, "No description available for this module.")
+    
+    q.put(f"Console")
+    q.put(f" ")
+    q.put(description_text)
+    
+    process_console_queue()
+    process_fig_queue()
+    create_menu_bar(parent)
+    after_id = parent_window.after(uppdate_frequency, lambda: main_thread_update_function(parent_window, q, fig_queue, canvas_widget))
+    parent_window.after_tasks.append(after_id)
 
     print("Root initialization complete")
     return parent_frame, vars_dict