spacr 0.3.2__py3-none-any.whl → 0.3.22__py3-none-any.whl

spacr/core.py

@@ -844,4 +844,108 @@ def generate_mediar_masks(src, settings, object_type):
         gc.collect()
         torch.cuda.empty_cache()
 
-    print("Mask generation completed.")
+    print("Mask generation completed.")
+
+def generate_screen_graphs(settings):
+    """
+    Generate screen graphs for different measurements in a given source directory.
+
+    Args:
+        src (str or list): Path(s) to the source directory or directories.
+        tables (list): List of tables to include in the analysis (default: ['cell', 'nucleus', 'pathogen', 'cytoplasm']).
+        graph_type (str): Type of graph to generate (default: 'bar').
+        summary_func (str or function): Function to summarize data (default: 'mean').
+        y_axis_start (float): Starting value for the y-axis (default: 0).
+        error_bar_type (str): Type of error bar to use ('std' or 'sem') (default: 'std').
+        theme (str): Theme for the graph (default: 'pastel').
+        representation (str): Representation for grouping (default: 'well').
+        
+    Returns:
+        figs (list): List of generated figures.
+        results (list): List of corresponding result DataFrames.
+    """
+    
+    from .plot import spacrGraph
+    from .io import _read_and_merge_data
+    from.utils import annotate_conditions
+
+    if isinstance(settings['src'], str):
+        srcs = [settings['src']]
+    else:
+        srcs = settings['src']
+
+    all_df = pd.DataFrame()
+    figs = []
+    results = []
+
+    for src in srcs:
+        db_loc = [os.path.join(src, 'measurements', 'measurements.db')]
+        
+        # Read and merge data from the database
+        df, _ = _read_and_merge_data(db_loc, settings['tables'], verbose=True, nuclei_limit=settings['nuclei_limit'], pathogen_limit=settings['pathogen_limit'], uninfected=settings['uninfected'])
+        
+        # Annotate the data
+        df = annotate_conditions(df, cells=settings['cells'], cell_loc=None, pathogens=settings['controls'], pathogen_loc=settings['controls_loc'], treatments=None, treatment_loc=None)
+        
+        # Calculate recruitment metric
+        df['recruitment'] = df['pathogen_channel_1_mean_intensity'] / df['cytoplasm_channel_1_mean_intensity']
+                
+        # Combine with the overall DataFrame
+        all_df = pd.concat([all_df, df], ignore_index=True)
+    
+        # Generate individual plot
+        plotter = spacrGraph(df,
+                             grouping_column='pathogen',
+                             data_column='recruitment',
+                             graph_type=settings['graph_type'],
+                             summary_func=settings['summary_func'],
+                             y_axis_start=settings['y_axis_start'],
+                             error_bar_type=settings['error_bar_type'],
+                             theme=settings['theme'],
+                             representation=settings['representation'])
+
+        plotter.create_plot()
+        fig = plotter.get_figure()
+        results_df = plotter.get_results()
+        
+        # Append to the lists
+        figs.append(fig)
+        results.append(results_df)
+    
+    # Generate plot for the combined data (all_df)
+    plotter = spacrGraph(all_df,
+                         grouping_column='pathogen',
+                         data_column='recruitment',
+                         graph_type=settings['graph_type'],
+                         summary_func=settings['summary_func'],
+                         y_axis_start=settings['y_axis_start'],
+                         error_bar_type=settings['error_bar_type'],
+                         theme=settings['theme'],
+                         representation=settings['representation'])
+
+    plotter.create_plot()
+    fig = plotter.get_figure()
+    results_df = plotter.get_results()
+    
+    figs.append(fig)
+    results.append(results_df)
+    
+    # Save figures and results
+    for i, fig in enumerate(figs):
+        res = results[i]
+        
+        if i < len(srcs):
+            source = srcs[i]
+        else:
+            source = srcs[0]
+
+        # Ensure the destination folder exists
+        dst = os.path.join(source, 'results')
+        print(f"Savings results to {dst}")
+        os.makedirs(dst, exist_ok=True)
+        
+        # Save the figure and results DataFrame
+        fig.savefig(os.path.join(dst, f"figure_controls_{i}_{settings['representation']}_{settings['summary_func']}_{settings['graph_type']}.pdf"), format='pdf')
+        res.to_csv(os.path.join(dst, f"results_controls_{i}_{settings['representation']}_{settings['summary_func']}_{settings['graph_type']}.csv"), index=False)
+
+    return

spacr/deep_spacr.py

@@ -1,4 +1,4 @@
-import os, torch, time, gc, datetime
+import os, torch, time, gc, datetime, cv2
 torch.backends.cudnn.benchmark = True
 
 import numpy as np
@@ -10,6 +10,8 @@ import torch.nn.functional as F
 import matplotlib.pyplot as plt
 from PIL import Image
 from sklearn.metrics import auc, precision_recall_curve
+from IPython.display import display
+from multiprocessing import cpu_count
 
 from torchvision import transforms
 from torch.utils.data import DataLoader
@@ -73,6 +75,12 @@ def apply_model_to_tar(settings={}):
     
     from .io import TarImageDataset
     from .utils import process_vision_results, print_progress
+
+    if os.path.exists(settings['dataset']):
+        tar_path = settings['dataset']
+    else:
+        tar_path = os.path.join(settings['src'], 'datasets', settings['dataset'])
+    model_path = settings['model_path']
     
     device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
     if settings['normalize']:
@@ -86,18 +94,18 @@ def apply_model_to_tar(settings={}):
             transforms.CenterCrop(size=(settings['image_size'], settings['image_size']))])
     
     if settings['verbose']:
-        print(f"Loading model from {settings['model_path']}")
-        print(f"Loading dataset from {settings['tar_path']}")
+        print(f"Loading model from {model_path}")
+        print(f"Loading dataset from {tar_path}")
         
     model = torch.load(settings['model_path'])
     
-    dataset = TarImageDataset(settings['tar_path'], transform=transform)
+    dataset = TarImageDataset(tar_path, transform=transform)
     data_loader = DataLoader(dataset, batch_size=settings['batch_size'], shuffle=True, num_workers=settings['n_jobs'], pin_memory=True)
     
-    model_name = os.path.splitext(os.path.basename(settings['model_path']))[0] 
-    dataset_name = os.path.splitext(os.path.basename(settings['tar_path']))[0]  
+    model_name = os.path.splitext(os.path.basename(model_path))[0]
+    dataset_name = os.path.splitext(os.path.basename(settings['dataset']))[0]  
     date_name = datetime.date.today().strftime('%y%m%d')
-    dst = os.path.dirname(settings['tar_path'])
+    dst = os.path.dirname(tar_path)
     result_loc = f'{dst}/{date_name}_{dataset_name}_{model_name}_result.csv'
 
     model.eval()
@@ -240,7 +248,7 @@ def evaluate_model_performance(model, loader, epoch, loss_type):
     
     loss /= len(loader)
     data_dict = classification_metrics(all_labels, prediction_pos_probs)
-    data_dict['loss'] = loss
+    data_dict['loss'] = loss.item()
     data_dict['epoch'] = epoch
     data_dict['Accuracy'] = acc
     
@@ -323,8 +331,8 @@ def test_model_performance(loaders, model, loader_name_list, epoch, loss_type):
 
 def train_test_model(settings):
     
-    from .io import _save_settings, _copy_missclassified
-    from .utils import pick_best_model
+    from .io import _copy_missclassified
+    from .utils import pick_best_model, save_settings
     from .io import generate_loaders
     from .settings import get_train_test_model_settings
 
@@ -346,7 +354,12 @@ def train_test_model(settings):
         model = torch.load(settings['custom_model_path'])
     
     if settings['train']:
-        _save_settings(settings, src)
+        if settings['train'] and settings['test']:
+            save_settings(settings, name=f"train_test_{settings['model_type']}_{settings['epochs']}", show=True)
+        elif settings['train'] is True:
+            save_settings(settings, name=f"train_{settings['model_type']}_{settings['epochs']}", show=True)
+        elif settings['test'] is True:
+            save_settings(settings, name=f"test_{settings['model_type']}_{settings['epochs']}", show=True)
 
     if settings['train']:
         train, val, train_fig  = generate_loaders(src, 
@@ -574,19 +587,21 @@ def train_model(dst, model_type, train_loaders, epochs=100, learning_rate=0.0001
             if schedule == 'step_lr':
                 scheduler.step()
 
-        if epoch % 10 == 0 or epoch == epochs:
-            if accumulated_train_dicts:
-                train_df = pd.DataFrame(accumulated_train_dicts)
-                _save_progress(dst, train_df, result_type='train')
-                
-            if accumulated_val_dicts:
-                val_df = pd.DataFrame(accumulated_val_dicts)
-                _save_progress(dst, val_df,result_type='validation')
-                
-            if accumulated_test_dicts:
-                val_df = pd.DataFrame(accumulated_test_dicts)
-                _save_progress(dst, val_df, result_type='test')
-                
+        if accumulated_train_dicts and accumulated_val_dicts:
+            train_df = pd.DataFrame(accumulated_train_dicts)
+            validation_df = pd.DataFrame(accumulated_val_dicts)
+            _save_progress(dst, train_df, validation_df)
+            accumulated_train_dicts, accumulated_val_dicts = [], []
+
+        elif accumulated_train_dicts:
+            train_df = pd.DataFrame(accumulated_train_dicts)
+            _save_progress(dst, train_df, None)
+            accumulated_train_dicts = []
+        elif accumulated_test_dicts:
+            test_df = pd.DataFrame(accumulated_test_dicts)
+            _save_progress(dst, test_df, None)
+            accumulated_test_dicts = []
+            
         batch_size = len(train_loaders)
         duration = time.time() - start_time
         time_ls.append(duration)
@@ -595,7 +610,138 @@ def train_model(dst, model_type, train_loaders, epochs=100, learning_rate=0.0001
             
     return model, model_path
 
-def visualize_saliency_map(src, model_type='maxvit', model_path='', image_size=224, channels=[1,2,3], normalize=True, class_names=None, save_saliency=False, save_dir='saliency_maps'):
+def visualize_saliency_map(settings):
+    from spacr.utils import SaliencyMapGenerator, print_progress
+    from spacr.io import TarImageDataset  # Assuming you have a dataset class
+    from torchvision.utils import make_grid
+
+    use_cuda = torch.cuda.is_available()
+    device = torch.device("cuda" if use_cuda else "cpu")
+
+    # Set number of jobs for loading
+    if settings['n_jobs'] is None:
+        n_jobs = max(1, cpu_count() - 4)
+    else:
+        n_jobs = settings['n_jobs']
+
+    # Set transforms for images
+    if settings['normalize']:
+        transform = transforms.Compose([
+            transforms.ToTensor(),
+            transforms.CenterCrop(size=(settings['image_size'], settings['image_size'])),
+            transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))])
+    else:
+        transform = transforms.Compose([
+            transforms.ToTensor(),
+            transforms.CenterCrop(size=(settings['image_size'], settings['image_size']))])
+
+    # Handle dataset path
+    if os.path.exists(settings['dataset']):
+        tar_path = settings['dataset']
+    else:
+        print(f"Dataset not found at {settings['dataset']}")
+        return
+    
+    if settings.get('save', False):
+        if settings['dtype'] not in ['uint8', 'uint16']:
+            print("Invalid dtype in settings. Please use 'uint8' or 'uint16'.")
+            return
+
+    # Load the model
+    model = torch.load(settings['model_path'])
+    model.to(device)
+    model.eval()  # Ensure the model is in evaluation mode
+
+    # Create directory for saving saliency maps if it does not exist
+    if settings.get('save', False):
+        dataset_dir = os.path.dirname(tar_path)
+        dataset_name = os.path.splitext(os.path.basename(tar_path))[0]
+        save_dir = os.path.join(dataset_dir, dataset_name, 'saliency_maps')
+        os.makedirs(save_dir, exist_ok=True)
+        print(f"Saliency maps will be saved in: {save_dir}")
+
+    # Load dataset
+    dataset = TarImageDataset(tar_path, transform=transform)
+    data_loader = DataLoader(dataset, batch_size=settings['batch_size'], shuffle=True, num_workers=n_jobs, pin_memory=True)
+
+    # Initialize SaliencyMapGenerator
+    cam_generator = SaliencyMapGenerator(model)
+    time_ls = []
+
+    for batch_idx, (inputs, filenames) in enumerate(data_loader):
+        start = time.time()
+        inputs = inputs.to(device)
+        
+        saliency_maps, predicted_classes = cam_generator.compute_saliency_and_predictions(inputs)
+
+        if settings['saliency_mode'] not in ['mean', 'sum']:
+            print("To generate channel average or sum saliency maps set saliency_mode to 'mean' or 'sum', respectively.")
+
+        if settings['saliency_mode'] == 'mean':
+            saliency_maps = saliency_maps.mean(dim=1, keepdim=True)
+
+        elif settings['saliency_mode'] == 'sum':
+            saliency_maps = saliency_maps.sum(dim=1, keepdim=True)
+
+        # Example usage with the class
+        if settings.get('plot', False):
+            if settings['plot_mode'] not in ['mean', 'channel', '3-channel']:
+                print("Invalid plot_mode in settings. Please use 'mean', 'channel', or '3-channel'.")
+                return
+            else:
+                cam_generator.plot_saliency_grid(inputs, saliency_maps, predicted_classes, mode=settings['plot_mode'])
+
+        if settings.get('save', False):
+            for i in range(inputs.size(0)):
+                saliency_map = saliency_maps[i].detach().cpu().numpy()
+
+                # Check dtype in settings and normalize accordingly
+                if settings['dtype'] == 'uint16':
+                    saliency_map = np.clip(saliency_map, 0, 1) * 65535
+                    saliency_map = saliency_map.astype(np.uint16)
+                    mode = 'I;16'
+                elif settings['dtype'] == 'uint8':
+                    saliency_map = np.clip(saliency_map, 0, 1) * 255
+                    saliency_map = saliency_map.astype(np.uint8)
+                    mode = 'L'  # Grayscale mode for uint8
+
+                # Get the class prediction (0 or 1)
+                class_pred = predicted_classes[i].item()
+
+                save_class_dir = os.path.join(save_dir, f'class_{class_pred}')
+                os.makedirs(save_class_dir, exist_ok=True)
+                save_path = os.path.join(save_class_dir, filenames[i])
+
+                # Handle different cases based on saliency_map dimensions
+                if saliency_map.ndim == 3:  # Multi-channel case (C, H, W)
+                    if saliency_map.shape[0] == 3:  # RGB-like saliency map
+                        saliency_image = Image.fromarray(np.moveaxis(saliency_map, 0, -1), mode="RGB")  # Convert (C, H, W) to (H, W, C)
+                    elif saliency_map.shape[0] == 1:  # Single-channel case (1, H, W)
+                        saliency_map = np.squeeze(saliency_map)  # Remove the extra channel dimension
+                        saliency_image = Image.fromarray(saliency_map, mode=mode)  # Use grayscale mode for single-channel
+                    else:
+                        raise ValueError(f"Unexpected number of channels: {saliency_map.shape[0]}")
+
+                elif saliency_map.ndim == 2:  # Single-channel case (H, W)
+                    saliency_image = Image.fromarray(saliency_map, mode=mode)  # Keep single channel (H, W)
+
+                else:
+                    raise ValueError(f"Unexpected number of dimensions: {saliency_map.ndim}")
+
+                # Save the image
+                saliency_image.save(save_path)
+
+
+        stop = time.time()
+        duration = stop - start
+        time_ls.append(duration)
+        files_processed = batch_idx * settings['batch_size']
+        files_to_process = len(data_loader)
+        print_progress(files_processed, files_to_process, n_jobs=n_jobs, time_ls=time_ls, batch_size=settings['batch_size'], operation_type="Generating Saliency Maps")
+
+    print("Saliency map generation complete.")
+
+def visualize_saliency_map_v1(src, model_type='maxvit', model_path='', image_size=224, channels=[1,2,3], normalize=True, class_names=None, save_saliency=False, save_dir='saliency_maps'):
 
     from spacr.utils import SaliencyMapGenerator, preprocess_image