MEDfl 0.2.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

Medfl/LearningManager/model.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+# froked from https://github.com/pythonlessons/mltu/blob/main/mltu/torch/model.py
+
+import typing
+from collections import OrderedDict
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+import torch
+import torch.nn as nn
+from sklearn.metrics import accuracy_score,roc_auc_score
+
+from .utils import params
+
+
+class Model:
+    """
+    Model class for training and testing PyTorch neural networks.
+
+    Attributes:
+        model (torch.nn.Module): PyTorch neural network.
+        optimizer (torch.optim.Optimizer): PyTorch optimizer.
+        criterion (typing.Callable): Loss function.
+    """
+
+    def __init__(
+        self,
+        model: torch.nn.Module,
+        optimizer: torch.optim.Optimizer,
+        criterion: typing.Callable,
+    ) -> None:
+        """
+        Initialize Model class with the specified model, optimizer, and criterion.
+
+        Args:
+            model (torch.nn.Module): PyTorch neural network.
+            optimizer (torch.optim.Optimizer): PyTorch optimizer.
+            criterion (typing.Callable): Loss function.
+        """
+        self.model = model
+        self.optimizer = optimizer
+        self.criterion = criterion
+        # Get device on which model is running
+        self.validate()
+
+    def validate(self) -> None:
+        """
+        Validate model and optimizer.
+        """
+        if not isinstance(self.model, torch.nn.Module):
+            raise TypeError("model argument must be a torch.nn.Module")
+
+        if not isinstance(self.optimizer, torch.optim.Optimizer):
+            raise TypeError(
+                "optimizer argument must be a torch.optim.Optimizer"
+            )
+
+    def get_parameters(self) -> List[np.ndarray]:
+        """
+        Get the parameters of the model as a list of NumPy arrays.
+
+        Returns:
+            List[np.ndarray]: The parameters of the model as a list of NumPy arrays.
+        """
+        return [
+            val.cpu().numpy() for _, val in self.model.state_dict().items()
+        ]
+
+    def set_parameters(self, parameters: List[np.ndarray]) -> None:
+        """
+        Set the parameters of the model from a list of NumPy arrays.
+
+        Args:
+            parameters (List[np.ndarray]): The parameters to be set.
+        """
+        params_dict = zip(self.model.state_dict().keys(), parameters)
+        state_dict = OrderedDict({k: torch.Tensor(v) for k, v in params_dict})
+        self.model.load_state_dict(state_dict, strict=True)
+
+    def train(
+        self, train_loader, epoch, device, privacy_engine, diff_priv=False
+    ) -> float:
+        """
+        Train the model on the given train_loader for one epoch.
+
+        Args:
+            train_loader: The data loader for training data.
+            epoch (int): The current epoch number.
+            device: The device on which to perform the training.
+            privacy_engine: The privacy engine used for differential privacy (if enabled).
+            diff_priv (bool, optional): Whether differential privacy is used. Default is False.
+
+        Returns:
+            float: The value of epsilon used in differential privacy.
+        """
+        self.model.train()
+        epsilon = 0
+        losses = []
+        top1_acc = []
+
+        for i, (X_train, y_train) in enumerate(train_loader):
+            X_train, y_train = X_train.to(device), y_train.to(device)  
+            
+            self.optimizer.zero_grad()
+
+            # compute output
+            y_hat = torch.squeeze(self.model(X_train), 1)
+            loss = self.criterion(y_hat, y_train)
+
+            preds = np.argmax(y_hat.detach().cpu().numpy(), axis=0)
+            labels = y_train.detach().cpu().numpy()
+
+            # measure accuracy and record loss
+            acc = (preds == labels).mean()
+
+            losses.append(loss.item())
+            top1_acc.append(acc)
+
+            loss.backward()
+            self.optimizer.step()
+
+            if diff_priv:
+                epsilon = privacy_engine.get_epsilon(float(params["DELTA"]))
+
+            if (i + 1) % 10 == 0:
+                if diff_priv:
+                    epsilon = privacy_engine.get_epsilon(float(params["DELTA"]))
+                    print(
+                        f"\tTrain Epoch: {epoch} \t"
+                        f"Loss: {np.mean(losses):.6f} "
+                        f"Acc@1: {np.mean(top1_acc) * 100:.6f} "
+                        f"(ε = {epsilon:.2f}, δ = {params['DELTA']})"
+                    )
+                else:
+                    print(
+                        f"\tTrain Epoch: {epoch} \t"
+                        f"Loss: {np.mean(losses):.6f} "
+                        f"Acc@1: {np.mean(top1_acc) * 100:.6f}"
+                    )
+
+        return epsilon
+
+    def evaluate(self, val_loader, device=torch.device("cpu")) -> Tuple[float, float]:
+        """
+        Evaluate the model on the given validation data.
+
+        Args:
+            val_loader: The data loader for validation data.
+            device: The device on which to perform the evaluation. Default is 'cpu'.
+
+        Returns:
+            Tuple[float, float]: The evaluation loss and accuracy.
+        """
+        correct, total, loss, accuracy, auc = 0, 0, 0.0, [], []
+        self.model.eval()
+
+        with torch.no_grad():
+            for X_test, y_test in val_loader:
+                X_test, y_test = X_test.to(device), y_test.to(device)  # Move data to device
+
+                y_hat = torch.squeeze(self.model(X_test), 1)
+               
+                
+                criterion = self.criterion.to(y_hat.device)
+                loss += criterion(y_hat, y_test).item()
+
+
+                # Move y_hat to CPU for accuracy computation
+                y_hat_cpu = y_hat.cpu().detach().numpy()
+                accuracy.append(accuracy_score(y_test.cpu().numpy(), y_hat_cpu.round()))
+
+                # Move y_test to CPU for AUC computation
+                y_test_cpu = y_test.cpu().numpy()
+                y_prob_cpu = y_hat.cpu().detach().numpy()
+                if (len(np.unique(y_test_cpu)) != 1):
+                    auc.append(roc_auc_score(y_test_cpu, y_prob_cpu))
+
+                total += y_test.size(0)
+                correct += np.sum(y_hat_cpu.round() == y_test_cpu)
+
+        loss /= len(val_loader.dataset)
+        return loss, np.mean(accuracy), np.mean(auc)
+
+
+    @staticmethod
+    def save_model(model , model_name:str):
+        """
+        Saves a PyTorch model to a file.
+
+        Args:
+            model (torch.nn.Module): PyTorch model to be saved.
+            model_name (str): Name of the model file.
+
+        Raises:
+            Exception: If there is an issue during the saving process.
+
+        Returns:
+            None
+        """
+        try:
+            torch.save(model, '../../notebooks/.ipynb_checkpoints/trainedModels/' + model_name + ".pth")
+        except Exception as e:
+            raise Exception(f"Error saving the model: {str(e)}")
+    
+    @staticmethod
+    def load_model(model_path: str):
+        """
+        Loads a PyTorch model from a file.
+
+        Args:
+            model_path (str): Path to the model file to be loaded.
+
+        Returns:
+            torch.nn.Module: Loaded PyTorch model.
+        """
+        # Ensure models are loaded onto the CPU when CUDA is not available
+        if torch.cuda.is_available():
+            loaded_model = torch.load(model_path)
+        else:
+            loaded_model = torch.load(model_path, map_location=torch.device('cpu'))
+        return loaded_model
+
+

Medfl/LearningManager/params.yaml

@@ -0,0 +1,14 @@
+DELTA: 1.0e-05
+EPSILON: 5.0
+MAX_GRAD_NORM: 1.0
+diff_privacy: true
+lr: 0.01
+min_evalclient: 2
+num_rounds: 12
+optimizer: SGD
+path_to_master_csv: /home/local/USHERBROOKE/saho6810/MEDfl/code/MEDfl/notebooks/data/masterDataSet/Mimic_ouael.csv
+path_to_test_csv: /home/local/USHERBROOKE/saho6810/MEDfl/code/MEDfl/notebooks/data/masterDataSet/Mimic_train.csv
+task: BinaryClassification
+test_batch_size: 1
+train_batch_size: 32
+train_epochs: 116

Medfl/LearningManager/params_optimiser.py

@@ -0,0 +1,442 @@
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import torch.nn.functional as F
+from torch.utils.data import TensorDataset, DataLoader
+from sklearn.model_selection import GridSearchCV, train_test_split
+from sklearn.base import BaseEstimator
+from sklearn.metrics import make_scorer, precision_score, recall_score, accuracy_score, f1_score,roc_auc_score, balanced_accuracy_score
+import optuna
+
+from MEDfl.LearningManager.model import Model
+from MEDfl.LearningManager.strategy import Strategy
+from MEDfl.LearningManager.server import FlowerServer
+from MEDfl.LearningManager.flpipeline import FLpipeline
+
+class BinaryClassifier(nn.Module):
+    def __init__(self, input_size, num_layers, layer_size):
+        super(BinaryClassifier, self).__init__()
+
+        # Input layer
+        self.layers = [nn.Linear(input_size, layer_size)]
+        
+        # Hidden layers
+        for _ in range(num_layers - 1):
+            self.layers.append(nn.Linear(layer_size, layer_size))
+        
+        # Output layer
+        self.layers.append(nn.Linear(layer_size, 1))
+        
+        # ModuleList to handle dynamic number of layers
+        self.layers = nn.ModuleList(self.layers)
+
+    def forward(self, x):
+        for layer in self.layers[:-1]:
+            x = F.relu(layer(x))
+        x = self.layers[-1](x)
+        return x
+    
+class CustomPyTorchClassifier(BaseEstimator):
+    def __init__(self, hidden_dim=10, lr=0.001, pos_weight=1, th=0.5, max_epochs=10, batch_size=32):
+        self.hidden_dim = hidden_dim
+        self.lr = lr
+        self.pos_weight = pos_weight
+        self.max_epochs = max_epochs
+        self.batch_size = batch_size
+        self.th = th
+        self.model = None
+
+    def fit(self, X, y):
+        if isinstance(X, torch.Tensor):
+            X = X.numpy()
+        if isinstance(y, torch.Tensor):
+            y = y.numpy()
+
+        input_dim = X.shape[1]
+        self.model = nn.Sequential(
+            nn.Linear(input_dim, self.hidden_dim),
+            nn.ReLU(),
+            nn.Linear(self.hidden_dim, 1),
+            nn.Sigmoid()
+        )
+
+        criterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(self.pos_weight))
+        optimizer = optim.Adam(self.model.parameters(), lr=self.lr)
+
+        train_data = TensorDataset(torch.from_numpy(X).float(), torch.from_numpy(y).float())
+        train_loader = DataLoader(train_data, batch_size=self.batch_size, shuffle=True)
+
+        self.model.train()
+        for epoch in range(self.max_epochs):
+            for inputs, labels in train_loader:
+                optimizer.zero_grad()
+                outputs = self.model(inputs)
+                loss = criterion(outputs.squeeze(), labels)
+                loss.backward()
+                optimizer.step()
+        return self
+
+    def predict(self, X):
+        if isinstance(X, torch.Tensor):
+            X = X.numpy()
+
+        self.model.eval()
+        with torch.no_grad():
+            outputs = self.model(torch.from_numpy(X).float())
+            predictions = (outputs.squeeze() > self.th).float().numpy()
+        return predictions
+
+    def score(self, X, y):
+        predictions = self.predict(X)
+        return accuracy_score(y, predictions)
+
+
+class ParamsOptimiser:
+    def __init__(self, X_train = None, y_train=None, X_test=None, y_test=None):
+        if isinstance(X_train, pd.DataFrame):
+            X_train = X_train.to_numpy()
+        if isinstance(y_train, pd.Series):
+            y_train = y_train.to_numpy()
+        if isinstance(X_test, pd.DataFrame):
+            X_test = X_test.to_numpy()
+        if isinstance(y_test, pd.Series):
+            y_test = y_test.to_numpy()
+
+        self.X_train = X_train
+        self.y_train = y_train
+        self.X_test = X_test
+        self.y_test = y_test
+
+    def perform_grid_search(self, param_grid, scoring_metric='recall', cv=3, verbose=1):
+        pytorch_model = CustomPyTorchClassifier()
+        scorer = make_scorer(recall_score, greater_is_better=True)
+
+        if scoring_metric == 'precision':
+            scorer = make_scorer(precision_score)
+        elif scoring_metric == 'accuracy':
+            scorer = make_scorer(accuracy_score)
+        elif scoring_metric == 'recall':
+            scorer = make_scorer(recall_score)
+        elif scoring_metric == 'f1':
+            scorer = make_scorer(f1_score)
+
+        grid_search = GridSearchCV(pytorch_model, param_grid, scoring=scorer, cv=cv, refit=scoring_metric, verbose=verbose)
+        grid_search.fit(self.X_train, self.y_train)
+
+        self.grid_search_results = grid_search  # Save the grid search results
+
+        return grid_search
+    
+    # Inside the CustomModelTrainer class
+    def plot_results(self, params_to_plot=None):
+        results = pd.DataFrame(self.grid_search_results.cv_results_)
+
+        if params_to_plot is None:
+            # Create a column for configuration details
+            results['config'] = results['params'].apply(lambda x: str(x))
+
+            # Visualize mean test scores along with configurations
+            plt.figure(figsize=(15, 8))
+            bar_plot = plt.bar(results.index, results['mean_test_score'], color='blue', alpha=0.7)
+            plt.xticks(results.index, results['config'], rotation='vertical', fontsize=8)
+            plt.ylabel('Mean Test Score')
+            plt.title('Mean Test Scores for Each Configuration')
+            plt.tight_layout()
+
+            # Add values on top of bars
+            for bar, score in zip(bar_plot, results['mean_test_score']):
+                plt.text(bar.get_x() + bar.get_width() / 2 - 0.15, bar.get_height() + 0.01, f'{score:.3f}', fontsize=8)
+
+            plt.show()
+            return 
+
+        try:
+            # Dynamically get the column names for the specified scoring metric
+            mean_test_col = f'mean_test_{params_to_plot[0]}'
+            param_cols = [f'param_{param}' for param in params_to_plot]
+
+            if len(params_to_plot) == 1:
+                # Plotting the heatmap for a single parameter
+                plt.figure(figsize=(8, 6))
+                sns.heatmap(results.pivot_table(index=param_cols[0]),
+                            annot=True, cmap='YlGnBu', fmt=".3f", cbar_kws={'label': mean_test_col})
+                plt.title(mean_test_col.capitalize())
+                plt.show()
+            elif len(params_to_plot) == 2:
+                # Create a pair plot for two parameters
+                plt.figure(figsize=(8, 6))
+                scores = results.pivot_table(index=param_cols[0], columns=param_cols[1], values=f'mean_test_score', aggfunc="mean")
+                sns.heatmap(scores, annot=True, cmap='YlGnBu', fmt=".3f", cbar_kws={'label': mean_test_col})
+                plt.title(mean_test_col.capitalize())
+                plt.show()
+            else:
+                print("Invalid number of parameters to plot. You can provide either one or two parameters.")
+        except KeyError as e:
+            print(f"Error: {e}. Make sure the specified scoring metric exists in the results DataFrame.")
+
+    
+
+    def optuna_optimisation(self, direction, params):
+        # Create the data loaders here
+        train_data = TensorDataset(torch.from_numpy(self.X_train).float(), torch.from_numpy(self.y_train).float())
+        test_data = TensorDataset(torch.from_numpy(self.X_test).float(), torch.from_numpy(self.y_test).float())
+
+       
+
+        def objective(trial):
+
+            batch_size=trial.suggest_int('batch_size', **params['batch_size'])
+
+            train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)
+            test_loader = DataLoader(test_data, batch_size=batch_size, shuffle=False)
+
+            # Create the model with the suggested hyperparameters
+            model = BinaryClassifier(input_size=self.X_train.shape[1],
+                                     num_layers=trial.suggest_int('num_layers', **params['num_layers']) ,
+                                     layer_size=trial.suggest_int('hidden_size', **params['hidden_size']))
+
+            # Define the loss function and optimizer
+            criterion = nn.BCEWithLogitsLoss()
+            optimizer_name = trial.suggest_categorical('optimizer', params['optimizer'])
+            learning_rate = trial.suggest_float('learning_rate', **params['learning_rate'])
+            
+
+            if optimizer_name == 'Adam':
+                optimizer = optim.Adam(model.parameters(), lr=learning_rate)
+            elif optimizer_name == 'SGD':
+                optimizer = optim.SGD(model.parameters(), lr=learning_rate)
+            elif optimizer_name == 'RMSprop':
+                optimizer = optim.RMSprop(model.parameters(), lr=learning_rate)
+
+            # Training loop
+            num_epochs = trial.suggest_int('num_epochs', **params['num_epochs'])
+            for epoch in range(num_epochs):
+                model.train()
+                for batch_X, batch_y in train_loader:
+                    optimizer.zero_grad()
+                    outputs = model(batch_X)
+                    loss = criterion(outputs.squeeze(), batch_y)
+                    loss.backward()
+                    optimizer.step()
+
+                # Evaluation
+                model.eval()
+                predictions = []
+                true_labels = []
+                with torch.no_grad():
+                    for batch_X, batch_y in test_loader:
+                        outputs = model(batch_X)
+                        predictions.extend(torch.sigmoid(outputs).numpy())
+                        true_labels.extend(batch_y.numpy())
+
+                # Calculate F1 score
+                # f1 = f1_score(true_labels, (np.array(predictions) > 0.5).astype(int))
+                auc = roc_auc_score(true_labels, predictions)
+
+                trial.report(auc, epoch)
+
+                # Handle pruning based on the intermediate value
+                if trial.should_prune():
+                    raise optuna.TrialPruned()
+
+            return auc
+
+        # Create an Optuna study
+        study = optuna.create_study(direction=direction)
+        study.optimize(objective, n_trials=params['n_trials'])
+        
+        self.study = study
+
+        # Get the best hyperparameters
+        best_params = study.best_params
+        print(f"Best Hyperparameters: {best_params}")
+
+        return study
+    
+    def train_optimized_model(self ,trial ,th_min , th_max):
+        
+        best_params = self.study.best_params
+
+        threshold = trial.suggest_float('threashhold', th_min, th_max, log=True)
+
+        train_data = TensorDataset(torch.from_numpy(self.X_train).float(), torch.from_numpy(self.y_train).float())
+        test_data = TensorDataset(torch.from_numpy(self.X_test).float(), torch.from_numpy(self.y_test).float())
+
+        train_loader = DataLoader(train_data, batch_size=best_params['batch_size'], shuffle=True)
+        test_loader = DataLoader(test_data, batch_size=best_params['batch_size'], shuffle=False)
+
+        
+        # Use the best hyperparameters to train the final model
+        final_model = BinaryClassifier(input_size=self.X_train.shape[1], layer_size=best_params['hidden_size'] , num_layers=best_params['num_layers'])
+        final_optimizer = self.get_optimizer(best_params['optimizer'], final_model.parameters(), best_params['learning_rate'])
+        final_criterion = nn.BCEWithLogitsLoss()
+
+        num_epochs = best_params['num_epochs']
+        for epoch in range(num_epochs):
+            final_model.train()
+            for batch_X, batch_y in train_loader:
+                final_optimizer.zero_grad()
+                outputs = final_model(batch_X)
+                loss = final_criterion(outputs.squeeze(), batch_y)
+                loss.backward()
+                final_optimizer.step()
+
+        # Evaluate the final model on the test set
+        final_model.eval()
+        with torch.no_grad():
+            predictions = []
+            true_labels = []
+            for batch_X, batch_y in test_loader:
+                outputs = final_model(batch_X)
+                predictions.extend(torch.sigmoid(outputs).numpy())
+                true_labels.extend(batch_y.numpy())
+
+        final_balanced_acc = balanced_accuracy_score(true_labels, (np.array(predictions) > threshold).astype(int))
+        print(f"Model balanced accuracy: {final_balanced_acc}")
+
+        return final_balanced_acc
+
+    def get_optimizer(self, optimizer_name, parameters, learning_rate):
+        if optimizer_name == 'Adam':
+            return optim.Adam(parameters, lr=learning_rate)
+        elif optimizer_name == 'SGD':
+            return optim.SGD(parameters, lr=learning_rate)
+        elif optimizer_name == 'RMSprop':
+            return optim.RMSprop(parameters, lr=learning_rate)
+        else:
+            raise ValueError(f"Unknown optimizer: {optimizer_name}")
+
+    def perform_grid_search(self, param_grid, scoring_metric='recall', cv=3, verbose=1):
+        pytorch_model = CustomPyTorchClassifier()
+        scorer = make_scorer(recall_score, greater_is_better=True)
+
+        if scoring_metric == 'precision':
+            scorer = make_scorer(precision_score)
+        elif scoring_metric == 'accuracy':
+            scorer = make_scorer(accuracy_score)
+        elif scoring_metric == 'recall':
+            scorer = make_scorer(recall_score)
+        elif scoring_metric == 'f1':
+            scorer = make_scorer(f1_score)
+
+        grid_search = GridSearchCV(pytorch_model, param_grid, scoring=scorer, cv=cv, refit=scoring_metric, verbose=verbose)
+        grid_search.fit(self.X_train, self.y_train)
+
+        self.grid_search_results = grid_search  # Save the grid search results
+
+        return grid_search
+    
+    
+    def plot_param_importances(self):
+        return optuna.visualization.plot_param_importances(self.study)
+    
+    def plot_slice(self , params):
+        return optuna.visualization.plot_slice(self.study , params=params)
+    
+    def plot_parallel_coordinate(self):
+        return optuna.visualization.plot_parallel_coordinate(self.study)
+    
+    def plot_rank(self , params=None):
+        return optuna.visualization.plot_rank(self.study , params=params)
+    
+    def plot_optimization_history(self):
+        return optuna.visualization.plot_optimization_history(self.study)
+    
+    def optimize_model_threashhold(self , n_trials , th_min , th_max):
+        additional_params = {'th_min': th_min, 'th_max': th_max}
+
+        th_study = optuna.create_study(direction='maximize')
+        th_study.optimize(lambda trial: self.train_optimized_model(trial , **additional_params) , n_trials)
+
+        # Get the best hyperparameters
+        best_params = th_study.best_params
+        print(f"Best Hyperparameters: {best_params}")
+
+        return optuna.visualization.plot_rank(th_study , params=['threashhold'])
+    
+    def federated_params_iptim(self , params , direction,  model, fl_dataset):
+
+        def objective(trial):
+          
+            criterion = nn.BCEWithLogitsLoss()
+
+            optimizer_name = trial.suggest_categorical('optimizer', params['optimizer'])
+            learning_rate = trial.suggest_float('learning_rate', **params['learning_rate'])
+            num_rounds = trial.suggest_int('num_rounds', **params['num_rounds'])
+            diff_privacy =  trial.suggest_int('diff_privacy', **params['diff_privacy'])
+            diff_privacy = True if diff_privacy == 1 else False
+
+            if optimizer_name == 'Adam':
+                optimizer = optim.Adam(model.parameters(), lr=learning_rate)
+            elif optimizer_name == 'SGD':
+                optimizer = optim.SGD(model.parameters(), lr=learning_rate)
+            elif optimizer_name == 'RMSprop':
+                optimizer = optim.RMSprop(model.parameters(), lr=learning_rate)
+
+            # Creating a new Model instance using the specific model created by DynamicModel
+            global_model = Model(model, optimizer, criterion)
+
+            # Get the initial params of the model 
+            init_params = global_model.get_parameters() 
+
+            fl_strategy = trial.suggest_categorical('fl_strategy', params['fl_strategy'])
+
+            learning_strategy = Strategy(fl_strategy, 
+                   fraction_fit = 1.0 ,
+                   fraction_evaluate = 1.0,
+                   min_fit_clients = 2,
+                   min_evaluate_clients = 2,
+                   min_available_clients = 2 , 
+                   initial_parameters=init_params)
+            
+            learning_strategy.create_strategy()
+
+            # Create The server 
+            server = FlowerServer(global_model, strategy = learning_strategy, num_rounds = num_rounds,
+                       num_clients  = len(fl_dataset.trainloaders), 
+                       fed_dataset = fl_dataset,diff_privacy = diff_privacy,
+                       # You can change the resources alocated for each client based on your machine 
+                       client_resources={'num_cpus': 1.0, 'num_gpus': 0.0}
+                       )
+            
+            ppl_1 = FLpipeline( name ="the first fl_pipeline",description = "this is our first FL pipeline",
+                   server = server)
+            
+            # Run the Traning of the model
+            history = ppl_1.server.run()
+
+            return server.auc[len(server.auc)-1]
+        
+
+       
+        study = optuna.create_study(direction=direction)
+        study.optimize(objective, n_trials=params['n_trials'])
+        
+        self.study = study
+
+        # Get the best hyperparameters
+        best_params = study.best_params
+        print(f"Best Hyperparameters: {best_params}")
+
+        return study
+
+
+        
+
+
+
+
+
+
+
+
+        
+    
+
+
+
+