spacr 0.2.53__py3-none-any.whl → 0.2.56__py3-none-any.whl

spacr/core.py

@@ -877,7 +877,106 @@ def annotate_results(pred_loc):
     display(df)
     return df
 
-def generate_dataset(src, file_metadata=None, experiment='TSG101_screen', sample=None):
+def generate_dataset(settings={}):
+    
+    from .utils import initiate_counter, add_images_to_tar
+    
+    db_path = os.path.join(settings['src'], 'measurements', 'measurements.db')
+    dst = os.path.join(settings['src'], 'datasets')
+    all_paths = []
+
+    # Connect to the database and retrieve the image paths
+    print(f"Reading DataBase: {db_path}")
+    try:
+        with sqlite3.connect(db_path) as conn:
+            cursor = conn.cursor()
+            if settings['file_metadata']:
+                if isinstance(settings['file_metadata'], str):
+                    cursor.execute("SELECT png_path FROM png_list WHERE png_path LIKE ?", (f"%{settings['file_metadata']}%",))
+            else:
+                cursor.execute("SELECT png_path FROM png_list")
+
+            while True:
+                rows = cursor.fetchmany(1000)
+                if not rows:
+                    break
+                all_paths.extend([row[0] for row in rows])
+
+    except sqlite3.Error as e:
+        print(f"Database error: {e}")
+        return
+    except Exception as e:
+        print(f"Error: {e}")
+        return
+
+    if isinstance(settings['sample'], int):
+        selected_paths = random.sample(all_paths, settings['sample'])
+        print(f"Random selection of {len(selected_paths)} paths")
+    else:
+        selected_paths = all_paths
+        random.shuffle(selected_paths)
+        print(f"All paths: {len(selected_paths)} paths")
+
+    total_images = len(selected_paths)
+    print(f"Found {total_images} images")
+
+    # Create a temp folder in dst
+    temp_dir = os.path.join(dst, "temp_tars")
+    os.makedirs(temp_dir, exist_ok=True)
+
+    # Chunking the data
+    num_procs = max(2, cpu_count() - 2)
+    chunk_size = len(selected_paths) // num_procs
+    remainder = len(selected_paths) % num_procs
+
+    paths_chunks = []
+    start = 0
+    for i in range(num_procs):
+        end = start + chunk_size + (1 if i < remainder else 0)
+        paths_chunks.append(selected_paths[start:end])
+        start = end
+
+    temp_tar_files = [os.path.join(temp_dir, f"temp_{i}.tar") for i in range(num_procs)]
+
+    print(f"Generating temporary tar files in {dst}")
+
+    # Initialize shared counter and lock
+    counter = Value('i', 0)
+    lock = Lock()
+
+    with Pool(processes=num_procs, initializer=initiate_counter, initargs=(counter, lock)) as pool:
+        pool.starmap(add_images_to_tar, [(paths_chunks[i], temp_tar_files[i], total_images) for i in range(num_procs)])
+
+    # Combine the temporary tar files into a final tar
+    date_name = datetime.date.today().strftime('%y%m%d')
+    if not settings['file_metadata'] is None:
+        tar_name = f"{date_name}_{settings['experiment']}_{settings['file_metadata']}.tar"
+    else:
+        tar_name = f"{date_name}_{settings['experiment']}.tar"
+    tar_name = os.path.join(dst, tar_name)
+    if os.path.exists(tar_name):
+        number = random.randint(1, 100)
+        tar_name_2 = f"{date_name}_{settings['experiment']}_{settings['file_metadata']}_{number}.tar"
+        print(f"Warning: {os.path.basename(tar_name)} exists, saving as {os.path.basename(tar_name_2)} ")
+        tar_name = os.path.join(dst, tar_name_2)
+
+    print(f"Merging temporary files")
+
+    with tarfile.open(tar_name, 'w') as final_tar:
+        for temp_tar_path in temp_tar_files:
+            with tarfile.open(temp_tar_path, 'r') as temp_tar:
+                for member in temp_tar.getmembers():
+                    file_obj = temp_tar.extractfile(member)
+                    final_tar.addfile(member, file_obj)
+            os.remove(temp_tar_path)
+
+    # Delete the temp folder
+    shutil.rmtree(temp_dir)
+    print(f"\nSaved {total_images} images to {tar_name}")
+
+    return tar_name
+
+def generate_dataset_v1(src, file_metadata=None, experiment='TSG101_screen', sample=None):
     
     from .utils import initiate_counter, add_images_to_tar
     
@@ -974,7 +1073,7 @@ def generate_dataset(src, file_metadata=None, experiment='TSG101_screen', sample
     shutil.rmtree(temp_dir)
     print(f"\nSaved {total_images} images to {tar_name}")
 
-def apply_model_to_tar(tar_path, model_path, file_type='cell_png', image_size=224, batch_size=64, normalize=True, preload='images', n_jobs=10, threshold=0.5, verbose=False):
+def apply_model_to_tar_v1(tar_path, model_path, file_type='cell_png', image_size=224, batch_size=64, normalize=True, preload='images', n_jobs=10, threshold=0.5, verbose=False):
     
     from .io import TarImageDataset
     from .utils import process_vision_results, print_progress
@@ -1044,6 +1143,76 @@ def apply_model_to_tar(tar_path, model_path, file_type='cell_png', image_size=22
     torch.cuda.memory.empty_cache()
     return df
 
+def apply_model_to_tar(settings={}):
+    
+    from .io import TarImageDataset
+    from .utils import process_vision_results, print_progress
+    
+    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+    if settings['normalize']:
+        transform = transforms.Compose([
+            transforms.ToTensor(),
+            transforms.CenterCrop(size=(settings['image_size'], settings['image_size'])),
+            transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))])
+    else:
+        transform = transforms.Compose([
+            transforms.ToTensor(),
+            transforms.CenterCrop(size=(settings['image_size'], settings['image_size']))])
+    
+    if settings['verbose']:
+        print(f"Loading model from {settings['model_path']}")
+        print(f"Loading dataset from {settings['tar_path']}")
+        
+    model = torch.load(settings['model_path'])
+    
+    dataset = TarImageDataset(settings['tar_path'], transform=transform)
+    data_loader = DataLoader(dataset, batch_size=settings['batch_size'], shuffle=True, num_workers=settings['n_jobs'], pin_memory=True)
+    
+    model_name = os.path.splitext(os.path.basename(settings['model_path']))[0] 
+    dataset_name = os.path.splitext(os.path.basename(settings['tar_path']))[0]  
+    date_name = datetime.date.today().strftime('%y%m%d')
+    dst = os.path.dirname(settings['tar_path'])
+    result_loc = f'{dst}/{date_name}_{dataset_name}_{model_name}_result.csv'
+
+    model.eval()
+    model = model.to(device)
+    
+    if settings['verbose']:
+        print(model)
+        print(f'Generated dataset with {len(dataset)} images')
+        print(f'Generating loader from {len(data_loader)} batches')
+        print(f'Results wil be saved in: {result_loc}')
+        print(f'Model is in eval mode')
+        print(f'Model loaded to device')
+        
+    prediction_pos_probs = []
+    filenames_list = []
+    time_ls = []
+    gc.collect()
+    with torch.no_grad():
+        for batch_idx, (batch_images, filenames) in enumerate(data_loader, start=1):
+            start = time.time()
+            images = batch_images.to(torch.float).to(device)
+            outputs = model(images)
+            batch_prediction_pos_prob = torch.sigmoid(outputs).cpu().numpy()
+            prediction_pos_probs.extend(batch_prediction_pos_prob.tolist())
+            filenames_list.extend(filenames)
+            stop = time.time()
+            duration = stop - start
+            time_ls.append(duration)
+            files_processed = batch_idx*settings['batch_size']
+            files_to_process = len(data_loader)
+            print_progress(files_processed, files_to_process, n_jobs=settings['n_jobs'], time_ls=time_ls, batch_size=settings['batch_size'], operation_type="Tar dataset")
+
+    data = {'path':filenames_list, 'pred':prediction_pos_probs}
+    df = pd.DataFrame(data, index=None)
+    df = process_vision_results(df, settings['score_threshold'])
+
+    df.to_csv(result_loc, index=True, header=True, mode='w')
+    torch.cuda.empty_cache()
+    torch.cuda.memory.empty_cache()
+    return df
+
 def apply_model(src, model_path, image_size=224, batch_size=64, normalize=True, n_jobs=10):
     
     from .io import NoClassDataset
@@ -1206,19 +1375,19 @@ def generate_dataset_from_lists(dst, class_data, classes, test_split=0.1):
         for path in train_data:
             start = time.time()
             shutil.copy(path, os.path.join(train_class_dir, os.path.basename(path)))
-            processed_files += 1
             duration = time.time() - start
             time_ls.append(duration)
             print_progress(processed_files, total_files, n_jobs=1, time_ls=None, batch_size=None, operation_type="Copying files for Train dataset")
+            processed_files += 1
 
         # Copy test files
         for path in test_data:
             start = time.time()
             shutil.copy(path, os.path.join(test_class_dir, os.path.basename(path)))
-            processed_files += 1
             duration = time.time() - start
             time_ls.append(duration)
             print_progress(processed_files, total_files, n_jobs=1, time_ls=None, batch_size=None, operation_type="Copying files for Test dataset")
+            processed_files += 1
 
     # Print summary
     for cls in classes:
@@ -1226,9 +1395,9 @@ def generate_dataset_from_lists(dst, class_data, classes, test_split=0.1):
         test_class_dir = os.path.join(dst, f'test/{cls}')
         print(f'Train class {cls}: {len(os.listdir(train_class_dir))}, Test class {cls}: {len(os.listdir(test_class_dir))}')
 
-    return
+    return os.path.join(dst, 'train'), os.path.join(dst, 'test')
 
-def generate_training_dataset(src, mode='annotation', annotation_column='test', annotated_classes=[1,2], classes=['nc','pc'], size=200, test_split=0.1, class_metadata=[['c1'],['c2']], metadata_type_by='col', channel_of_interest=3, custom_measurement=None, tables=None, png_type='cell_png'):
+def generate_training_dataset_v1(src, mode='annotation', annotation_column='test', annotated_classes=[1,2], classes=['nc','pc'], size=200, test_split=0.1, class_metadata=[['c1'],['c2']], metadata_type_by='col', channel_of_interest=3, custom_measurement=None, tables=None, png_type='cell_png'):
     
     from .io import _read_and_merge_data, _read_db
     from .utils import get_paths_from_db, annotate_conditions
@@ -1329,6 +1498,110 @@ def generate_training_dataset(src, mode='annotation', annotation_column='test',
     
     return
 
+def generate_training_dataset(settings):
+    
+    from .io import _read_and_merge_data, _read_db
+    from .utils import get_paths_from_db, annotate_conditions
+    from .settings import set_generate_training_dataset_defaults
+
+    settings = set_generate_training_dataset_defaults(settings)
+    
+    db_path = os.path.join(settings['src'], 'measurements','measurements.db')
+    dst = os.path.join(settings['src'], 'datasets', 'training')
+
+    if os.path.exists(dst):
+        for i in range(1, 1000):
+            dst = os.path.join(settings['src'], 'datasets', f'training_{i}')
+            if not os.path.exists(dst):
+                print(f'Creating new directory for training: {dst}')
+                break
+                
+    if settings['dataset_mode'] == 'annotation':
+        class_paths_ls_2 = []
+        class_paths_ls = training_dataset_from_annotation(db_path, dst, settings['annotation_column'], annotated_classes=settings['annotated_classes'])
+        for class_paths in class_paths_ls:
+            class_paths_temp = random.sample(class_paths, settings['size'])
+            class_paths_ls_2.append(class_paths_temp)
+        class_paths_ls = class_paths_ls_2
+
+    elif settings['dataset_mode'] == 'metadata':
+        class_paths_ls = []
+        class_len_ls = []
+        [df] = _read_db(db_loc=db_path, tables=['png_list'])
+        df['metadata_based_class'] = pd.NA
+        for i, class_ in enumerate(settings['classes']):
+            ls = settings['class_metadata'][i]
+            df.loc[df[settings['metadata_type_by']].isin(ls), 'metadata_based_class'] = class_
+            
+        for class_ in settings['classes']:
+            if settings['size'] == None:
+                c_s = []
+                for c in settings['classes']:
+                    c_s_t_df = df[df['metadata_based_class'] == c]
+                    c_s.append(len(c_s_t_df))
+                    print(f'Found {len(c_s_t_df)} images for class {c}')
+                size = min(c_s)
+                print(f'Using the smallest class size: {size}')
+
+            class_temp_df = df[df['metadata_based_class'] == class_]
+            class_len_ls.append(len(class_temp_df))
+            print(f'Found {len(class_temp_df)} images for class {class_}')
+            class_paths_temp = random.sample(class_temp_df['png_path'].tolist(), settings['size'])
+            class_paths_ls.append(class_paths_temp)
+    
+    elif settings['dataset_mode'] == 'recruitment':
+        class_paths_ls = []
+        if not isinstance(settings['tables'], list):
+            tables = ['cell', 'nucleus', 'pathogen','cytoplasm']
+        
+        df, _ = _read_and_merge_data(locs=[db_path],
+                                    tables=tables,
+                                    verbose=False,
+                                    include_multinucleated=True,
+                                    include_multiinfected=True,
+                                    include_noninfected=True)
+        
+        print('length df 1', len(df))
+        
+        df = annotate_conditions(df, cells=['HeLa'], cell_loc=None, pathogens=['pathogen'], pathogen_loc=None, treatments=settings['classes'], treatment_loc=settings['class_metadata'], types = settings['metadata_type_by'])
+        print('length df 2', len(df))
+        [png_list_df] = _read_db(db_loc=db_path, tables=['png_list'])
+	    
+        if settings['custom_measurement'] != None:
+        
+            if not isinstance(settings['custom_measurement'], list):
+                 print(f'custom_measurement should be a list, add [ measurement_1,  measurement_2 ] or [ measurement ]')
+                 return
+        	
+            if isinstance(settings['custom_measurement'], list):
+                if len(settings['custom_measurement']) == 2:
+                    print(f"Classes will be defined by the Q1 and Q3 quantiles of recruitment ({settings['custom_measurement'][0]}/{settings['custom_measurement'][1]})")
+                    df['recruitment'] = df[f"{settings['custom_measurement'][0]}']/df[f'{settings['custom_measurement'][1]}"]
+                if len(settings['custom_measurement']) == 1:
+                    print(f"Classes will be defined by the Q1 and Q3 quantiles of recruitment ({settings['custom_measurement'][0]})")
+                    df['recruitment'] = df[f"{settings['custom_measurement'][0]}"]
+        else:
+            print(f"Classes will be defined by the Q1 and Q3 quantiles of recruitment (pathogen/cytoplasm for channel {settings['channel_of_interest']})")
+            df['recruitment'] = df[f"pathogen_channel_{settings['channel_of_interest']}_mean_intensity']/df[f'cytoplasm_channel_{settings['channel_of_interest']}_mean_intensity"]
+		
+        q25 = df['recruitment'].quantile(0.25)
+        q75 = df['recruitment'].quantile(0.75)
+        df_lower = df[df['recruitment'] <= q25]
+        df_upper = df[df['recruitment'] >= q75]
+        
+        class_paths_lower = get_paths_from_db(df=df_lower, png_df=png_list_df, image_type=settings['png_type'])
+        
+        class_paths_lower = random.sample(class_paths_lower['png_path'].tolist(), settings['size'])
+        class_paths_ls.append(class_paths_lower)
+        
+        class_paths_upper = get_paths_from_db(df=df_upper, png_df=png_list_df, image_type=settings['png_type'])
+        class_paths_upper = random.sample(class_paths_upper['png_path'].tolist(), settings['size'])
+        class_paths_ls.append(class_paths_upper)
+    
+    train_class_dir, test_class_dir = generate_dataset_from_lists(dst, class_data=class_paths_ls, classes=settings['classes'], test_split=settings['test_split'])
+    
+    return train_class_dir, test_class_dir
+
 def generate_loaders(src, train_mode='erm', mode='train', image_size=224, batch_size=32, classes=['nc','pc'], n_jobs=None, validation_split=0.0, max_show=2, pin_memory=False, normalize=False, channels=[1, 2, 3], augment=False, verbose=False):
     
     """
@@ -2497,7 +2770,6 @@ def ml_analysis(df, channel_of_interest=3, location_column='col', positive_contr
     df_metadata = df[[location_column]].copy()
     df, features = filter_dataframe_features(df, channel_of_interest, exclude, remove_low_variance_features, remove_highly_correlated_features, verbose)
     
-    
     if verbose:
         print(f'Found {len(features)} numerical features in the dataframe')
         print(f'Features used in training: {features}')
@@ -2642,7 +2914,6 @@ def check_index(df, elements=5, split_char='_'):
             print(idx)
         raise ValueError(f"Found {len(problematic_indices)} problematic indices that do not split into {elements} parts.")
 
-#def plate_heatmap(src, model_type='xgboost', variable='predictions', grouping='mean', min_max='allq', cmap='viridis', channel_of_interest=3, min_count=25, n_estimators=100, col_to_compare='col', pos='c2', neg='c1', exclude=None, n_repeats=10, clean=True, nr_to_plot=20, verbose=False, n_jobs=-1):
 def generate_ml_scores(src, settings):
     
     from .io import _read_and_merge_data
@@ -2680,7 +2951,7 @@ def generate_ml_scores(src, settings):
                                settings['top_features'],
                                settings['n_estimators'],
                                settings['test_size'],
-                               settings['model_type'],
+                               settings['model_type_ml'],
                                settings['n_jobs'],
                                settings['remove_low_variance_features'],
                                settings['remove_highly_correlated_features'],
@@ -2701,7 +2972,7 @@ def generate_ml_scores(src, settings):
                                 min_count=settings['minimum_cell_count'],
                                 verbose=settings['verbose'])
 
-    data_path, permutation_path, feature_importance_path, model_metricks_path, permutation_fig_path, feature_importance_fig_path, shap_fig_path, plate_heatmap_path, settings_csv = get_ml_results_paths(src, settings['model_type'], settings['channel_of_interest'])
+    data_path, permutation_path, feature_importance_path, model_metricks_path, permutation_fig_path, feature_importance_fig_path, shap_fig_path, plate_heatmap_path, settings_csv = get_ml_results_paths(src, settings['model_type_ml'], settings['channel_of_interest'])
     df, permutation_df, feature_importance_df, _, _, _, _, _, metrics_df = output
 
     settings_df.to_csv(settings_csv, index=False)
@@ -2858,6 +3129,7 @@ def generate_image_umap(settings={}):
         settings['plot_outlines'] = False
         settings['smooth_lines'] = False
 
+    print(f'Generating Image UMAP ...')
     settings_df = pd.DataFrame(list(settings.items()), columns=['Key', 'Value'])
     settings_dir = os.path.join(settings['src'][0],'settings')
     settings_csv = os.path.join(settings_dir,'embedding_settings.csv')

spacr/deep_spacr.py

@@ -196,7 +196,7 @@ def test_model_performance(loaders, model, loader_name_list, epoch, train_mode,
     test_time = end_time - start_time
     return result, results_df
 
-def train_test_model(src, settings, custom_model=False, custom_model_path=None):
+def train_test_model(settings):
     
     from .io import _save_settings, _copy_missclassified
     from .utils import pick_best_model
@@ -208,7 +208,10 @@ def train_test_model(src, settings, custom_model=False, custom_model_path=None):
     gc.collect()
 
     settings = set_default_train_test_model(settings)
-    channels_str = ''.join(settings['channels'])
+
+    src = settings['src']
+
+    channels_str = ''.join(settings['train_channels'])
     dst = os.path.join(src,'model', settings['model_type'], channels_str, str(f"epochs_{settings['epochs']}"))
     os.makedirs(dst, exist_ok=True)
     settings['src'] = src
@@ -217,8 +220,8 @@ def train_test_model(src, settings, custom_model=False, custom_model_path=None):
     settings_csv = os.path.join(dst,'train_test_model_settings.csv')
     settings_df.to_csv(settings_csv, index=False)
     
-    if custom_model:
-        model = torch.load(custom_model_path)
+    if settings['custom_model']:
+        model = torch.load(settings['custom_model_path'])
     
     if settings['train']:
         _save_settings(settings, src)
@@ -234,7 +237,7 @@ def train_test_model(src, settings, custom_model=False, custom_model_path=None):
                                                     validation_split=settings['val_split'],
                                                     pin_memory=settings['pin_memory'],
                                                     normalize=settings['normalize'],
-                                                    channels=settings['channels'],
+                                                    channels=settings['train_channels'],
                                                     augment=settings['augment'],
                                                     verbose=settings['verbose'])
         
@@ -242,28 +245,28 @@ def train_test_model(src, settings, custom_model=False, custom_model_path=None):
         train_fig.savefig(train_batch_1_figure, format='pdf', dpi=600)
     
     if settings['train']:
-        model = train_model(dst = settings['dst'],
-                            model_type=settings['model_type'],
-                            train_loaders = train, 
-                            train_loader_names = plate_names, 
-                            train_mode = settings['train_mode'], 
-                            epochs = settings['epochs'], 
-                            learning_rate = settings['learning_rate'],
-                            init_weights = settings['init_weights'],
-                            weight_decay = settings['weight_decay'], 
-                            amsgrad = settings['amsgrad'], 
-                            optimizer_type = settings['optimizer_type'], 
-                            use_checkpoint = settings['use_checkpoint'], 
-                            dropout_rate = settings['dropout_rate'], 
-                            n_jobs = settings['n_jobs'], 
-                            val_loaders = val, 
-                            test_loaders = None, 
-                            intermedeate_save = settings['intermedeate_save'],
-                            schedule = settings['schedule'],
-                            loss_type=settings['loss_type'], 
-                            gradient_accumulation=settings['gradient_accumulation'], 
-                            gradient_accumulation_steps=settings['gradient_accumulation_steps'],
-                            channels=settings['channels'])
+        model, model_path = train_model(dst = settings['dst'],
+                                        model_type=settings['model_type'],
+                                        train_loaders = train, 
+                                        train_loader_names = plate_names, 
+                                        train_mode = settings['train_mode'], 
+                                        epochs = settings['epochs'], 
+                                        learning_rate = settings['learning_rate'],
+                                        init_weights = settings['init_weights'],
+                                        weight_decay = settings['weight_decay'], 
+                                        amsgrad = settings['amsgrad'], 
+                                        optimizer_type = settings['optimizer_type'], 
+                                        use_checkpoint = settings['use_checkpoint'], 
+                                        dropout_rate = settings['dropout_rate'], 
+                                        n_jobs = settings['n_jobs'], 
+                                        val_loaders = val, 
+                                        test_loaders = None, 
+                                        intermedeate_save = settings['intermedeate_save'],
+                                        schedule = settings['schedule'],
+                                        loss_type=settings['loss_type'], 
+                                        gradient_accumulation=settings['gradient_accumulation'], 
+                                        gradient_accumulation_steps=settings['gradient_accumulation_steps'],
+                                        channels=settings['train_channels'])
         
         torch.cuda.empty_cache()
         torch.cuda.memory.empty_cache()
@@ -280,7 +283,7 @@ def train_test_model(src, settings, custom_model=False, custom_model_path=None):
                                                      validation_split=0.0,
                                                      pin_memory=settings['pin_memory'],
                                                      normalize=settings['normalize'],
-                                                     channels=settings['channels'],
+                                                     channels=settings['train_channels'],
                                                      augment=False,
                                                      verbose=settings['verbose'])
         if model == None:
@@ -314,6 +317,8 @@ def train_test_model(src, settings, custom_model=False, custom_model_path=None):
     torch.cuda.empty_cache()
     torch.cuda.memory.empty_cache()
     gc.collect()
+
+    return model_path
     
 def train_model(dst, model_type, train_loaders, train_loader_names, train_mode='erm', epochs=100, learning_rate=0.0001, weight_decay=0.05, amsgrad=False, optimizer_type='adamw', use_checkpoint=False, dropout_rate=0, n_jobs=20, val_loaders=None, test_loaders=None, init_weights='imagenet', intermedeate_save=None, chan_dict=None, schedule = None, loss_type='binary_cross_entropy_with_logits', gradient_accumulation=False, gradient_accumulation_steps=4, channels=['r','g','b']):
     """
@@ -348,7 +353,7 @@ def train_model(dst, model_type, train_loaders, train_loader_names, train_mode='
     """    
     
     from .io import _save_model, _save_progress
-    from .utils import compute_irm_penalty, calculate_loss, choose_model
+    from .utils import compute_irm_penalty, calculate_loss, choose_model, print_progress
     
     print(f'Train batches:{len(train_loaders)}, Validation batches:{len(val_loaders)}')
     
@@ -386,6 +391,7 @@ def train_model(dst, model_type, train_loaders, train_loader_names, train_mode='
     else:
         scheduler = None
 
+    time_ls = []
     if train_mode == 'erm':
         for epoch in range(1, epochs+1):
             model.train()
@@ -412,7 +418,13 @@ def train_model(dst, model_type, train_loaders, train_loader_names, train_mode='
                     optimizer.zero_grad()
 
                 avg_loss = running_loss / batch_idx
-                print(f'\rTrain: epoch: {epoch} batch: {batch_idx}/{len(train_loaders)} avg_loss: {avg_loss:.5f} time: {(time.time()-start_time):.5f}', end='\r', flush=True)
+                #print(f'\rTrain: epoch: {epoch} batch: {batch_idx}/{len(train_loaders)} avg_loss: {avg_loss:.5f} time: {(time.time()-start_time):.5f}', end='\r', flush=True)
+                
+                batch_size = len(train_loaders)
+                duration = time.time() - start_time
+                time_ls.append(duration)
+                metricks = f"Loss: {avg_loss:.5f}"
+                print_progress(files_processed=epoch, files_to_process=epochs, n_jobs=1, time_ls=time_ls, batch_size=batch_size, operation_type=f"Training {model_type} model", metricks=metricks)
 
             end_time = time.time()
             train_time = end_time - start_time
@@ -421,6 +433,7 @@ def train_model(dst, model_type, train_loaders, train_loader_names, train_mode='
             train_names = 'train'
             results_df, train_test_time = evaluate_model_performance(train_loaders, model, train_names, epoch, train_mode='erm', loss_type=loss_type)
             train_metrics_df['train_test_time'] = train_test_time
+
             if val_loaders != None:
                 val_names = 'val'
                 result, val_time = evaluate_model_performance(val_loaders, model, val_names, epoch, train_mode='erm', loss_type=loss_type)
@@ -430,6 +443,7 @@ def train_model(dst, model_type, train_loaders, train_loader_names, train_mode='
                 
                 results_df = pd.concat([results_df, result])
                 train_metrics_df['val_time'] = val_time
+
             if test_loaders != None:
                 test_names = 'test'
                 result, test_test_time = evaluate_model_performance(test_loaders, model, test_names, epoch, train_mode='erm', loss_type=loss_type)
@@ -444,9 +458,30 @@ def train_model(dst, model_type, train_loaders, train_loader_names, train_mode='
                     scheduler.step()
             
             _save_progress(dst, results_df, train_metrics_df, epoch, epochs)
-            clear_output(wait=True)
-            display(results_df)
-            _save_model(model, model_type, results_df, dst, epoch, epochs, intermedeate_save=[0.99,0.98,0.95,0.94], channels=channels)
+            #clear_output(wait=True)
+            #display(results_df)
+
+            train_idx = f"{epoch}_train"
+            val_idx = f"{epoch}_val"
+            train_acc = results_df.loc[train_idx, 'accuracy']
+            neg_train_acc = results_df.loc[train_idx, 'neg_accuracy']
+            pos_train_acc = results_df.loc[train_idx, 'pos_accuracy']
+            val_acc = results_df.loc[val_idx, 'accuracy']
+            neg_val_acc = results_df.loc[val_idx, 'neg_accuracy']
+            pos_val_acc = results_df.loc[val_idx, 'pos_accuracy']
+            train_loss = results_df.loc[train_idx, 'loss']
+            train_prauc = results_df.loc[train_idx, 'prauc']
+            val_loss = results_df.loc[val_idx, 'loss']
+            val_prauc = results_df.loc[val_idx, 'prauc']
+
+            metricks = f"Train Acc: {train_acc:.5f} Val Acc: {val_acc:.5f} Train Loss: {train_loss:.5f} Val Loss: {val_loss:.5f} Train PRAUC: {train_prauc:.5f} Val PRAUC: {val_prauc:.5f}, Nc Train Acc: {neg_train_acc:.5f} Nc Val Acc: {neg_val_acc:.5f} Pc Train Acc: {pos_train_acc:.5f} Pc Val Acc: {pos_val_acc:.5f}"
+
+            batch_size = len(train_loaders)
+            duration = time.time() - start_time
+            time_ls.append(duration)
+            print_progress(files_processed=epoch, files_to_process=epochs, n_jobs=1, time_ls=time_ls, batch_size=batch_size, operation_type=f"Training {model_type} model", metricks=metricks)
+
+            model_path = _save_model(model, model_type, results_df, dst, epoch, epochs, intermedeate_save=[0.99,0.98,0.95,0.94], channels=channels)
             
     if train_mode == 'irm':
         dummy_w = torch.nn.Parameter(torch.Tensor([1.0])).to(device)
@@ -517,9 +552,10 @@ def train_model(dst, model_type, train_loaders, train_loader_names, train_mode='
             clear_output(wait=True)
             display(results_df)
             _save_progress(dst, results_df, train_metrics_df, epoch, epochs)
-            _save_model(model, model_type, results_df, dst, epoch, epochs, intermedeate_save=[0.99,0.98,0.95,0.94])
-            print(f'Saved model: {dst}')
-    return model
+            model_path = _save_model(model, model_type, results_df, dst, epoch, epochs, intermedeate_save=[0.99,0.98,0.95,0.94])
+            print(f'Saved model: {model_path}')
+    
+    return model, model_path
 
 def visualize_saliency_map(src, model_type='maxvit', model_path='', image_size=224, channels=[1,2,3], normalize=True, class_names=None, save_saliency=False, save_dir='saliency_maps'):
 
@@ -778,8 +814,32 @@ def visualize_smooth_grad(src, model_path, target_label_idx, image_size=224, cha
             smooth_grad_image = Image.fromarray((smooth_grad_map * 255).astype(np.uint8))
             smooth_grad_image.save(os.path.join(save_dir, f'smooth_grad_{file}'))
 
-# Usage
-#src = '/path/to/images'
-#model_path = '/path/to/model.pth'
-#target_label_idx = 0  # Change this to the target class index
-#visualize_smooth_grad(src, model_path, target_label_idx)
+def deep_spacr(settings={}):
+    from .settings import deep_spacr_defaults
+    from .core import generate_training_dataset, generate_dataset, apply_model_to_tar
+    
+    settings = deep_spacr_defaults(settings)
+    src = settings['src']
+    
+    if settings['train'] or settings['test']:
+        if settings['generate_training_dataset']:
+            print(f"Generating train and test datasets ...")
+            train_path, test_path = generate_training_dataset(settings)
+            print(f'Generated Train set: {train_path}')
+            print(f'Generated Train set: {test_path}')
+            settings['src'] = os.path.dirname(train_path)
+    
+    if settings['train_DL_model']:
+        print(f"Training model ...")
+        model_path = train_test_model(settings)
+        settings['model_path'] = model_path
+        settings['src'] = src
+        
+    if settings['apply_model_to_dataset']:
+        if not os.path.exists(settings['tar_path']):
+            print(f"Generating dataset ...")
+            tar_path = generate_dataset(settings)
+            settings['tar_path'] = tar_path
+            
+        if os.path.exists(settings['model_path']):
+            apply_model_to_tar(settings)

spacr/gui.py

@@ -27,7 +27,7 @@ class MainApp(tk.Tk):
         }
 
         self.additional_gui_apps = {
-            "Sequencing": (lambda frame: initiate_root(self, 'sequencing'), "Analyze sequencing data."),
+            #"Sequencing": (lambda frame: initiate_root(self, 'sequencing'), "Analyze sequencing data."),
             "Umap": (lambda frame: initiate_root(self, 'umap'), "Generate UMAP embeddings with datapoints represented as images."),
             "Train Cellpose": (lambda frame: initiate_root(self, 'train_cellpose'), "Train custom Cellpose models."),
             "ML Analyze": (lambda frame: initiate_root(self, 'ml_analyze'), "Machine learning analysis of data."),