MEDfl 2.0.4.dev0__py3-none-any.whl → 2.0.4.dev2__py3-none-any.whl

Medfl/LearningManager/server.py

@@ -1,181 +0,0 @@
-#!/usr/bin/env python3
-
-import copy
-from typing import Dict, Optional, Tuple
-
-import flwr as fl
-import torch
-
-from .client import FlowerClient
-from .federated_dataset import FederatedDataset
-from .model import Model
-from .strategy import Strategy
-
-
-class FlowerServer:
-    """
-    A class representing the central server for Federated Learning using Flower.
-
-    Attributes:
-        global_model (Model): The global model that will be federated among clients.
-        strategy (Strategy): The strategy used for federated learning, specifying communication and aggregation methods.
-        num_rounds (int): The number of federated learning rounds to perform.
-        num_clients (int): The number of clients participating in the federated learning process.
-        fed_dataset (FederatedDataset): The federated dataset used for training and evaluation.
-        diff_priv (bool): Whether differential privacy is used during the federated learning process.
-        accuracies (List[float]): A list to store the accuracy of the global model during each round.
-        losses (List[float]): A list to store the loss of the global model during each round.
-        flower_clients (List[FlowerClient]): A list to store the FlowerClient objects representing individual clients.
-     
-    """
-
-    def __init__(
-        self,
-        global_model: Model,
-        strategy: Strategy,
-        num_rounds: int,
-        num_clients: int,
-        fed_dataset: FederatedDataset,
-        diff_privacy: bool = False,
-        client_resources: Optional[Dict[str, float]] = {'num_cpus': 1, 'num_gpus': 0.0}
-    ) -> None:
-        """
-        Initialize a FlowerServer object with the specified parameters.
-
-        Args:
-            global_model (Model): The global model that will be federated among clients.
-            strategy (Strategy): The strategy used for federated learning, specifying communication and aggregation methods.
-            num_rounds (int): The number of federated learning rounds to perform.
-            num_clients (int): The number of clients participating in the federated learning process.
-            fed_dataset (FederatedDataset): The federated dataset used for training and evaluation.
-            diff_privacy (bool, optional): Whether differential privacy is used during the federated learning process.
-                                           Default is False.
-        """
-        self.device = torch.device(
-            f"cuda" if torch.cuda.is_available() else "cpu"
-        )
-        self.global_model = global_model
-        self.params = global_model.get_parameters()
-        self.global_model.model = global_model.model.to(self.device)
-        self.num_rounds = num_rounds
-        self.num_clients = num_clients
-        self.fed_dataset = fed_dataset
-        self.strategy = strategy
-        self.client_resources = client_resources
-        setattr(
-            self.strategy.strategy_object,
-            "min_available_clients",
-            self.num_clients,
-        )
-        setattr(
-            self.strategy.strategy_object,
-            "initial_parameters",
-            fl.common.ndarrays_to_parameters(self.params),
-        )
-        setattr(self.strategy.strategy_object, "evaluate_fn", self.evaluate)
-        self.fed_dataset = fed_dataset
-        self.diff_priv = diff_privacy
-        self.accuracies = []
-        self.losses = []
-        self.auc = []
-        self.flower_clients = []
-        self.validate()
-
-    def validate(self) -> None:
-        """Validate global_model, strategy, num_clients, num_rounds, fed_dataset, diff_privacy"""
-        if not isinstance(self.global_model, Model):
-            raise TypeError("global_model argument must be a Model instance")
-
-        # if not isinstance(self.strategy, Strategy):
-        #     print(self.strategy)
-        #     print(isinstance(self.strategy, Strategy))
-        #     raise TypeError("strategy argument must be a Strategy instance")
-
-        if not isinstance(self.num_clients, int):
-            raise TypeError("num_clients argument must be an int")
-
-        if not isinstance(self.num_rounds, int):
-            raise TypeError("num_rounds argument must be an int")
-
-        if not isinstance(self.diff_priv, bool):
-            raise TypeError("diff_priv argument must be a bool")
-
-    def client_fn(self, cid) -> FlowerClient:
-        """
-        Return a FlowerClient object for a specific client ID.
-
-        Args:
-            cid: The client ID.
-
-        Returns:
-            FlowerClient: A FlowerClient object representing the individual client.
-        """
-        
-        device = torch.device(
-            f"cuda:{int(cid) % 4}" if torch.cuda.is_available() else "cpu"
-        )
-        client_model = copy.deepcopy(self.global_model)
-      
-        trainloader = self.fed_dataset.trainloaders[int(cid)]
-        valloader = self.fed_dataset.valloaders[int(cid)]
-        # this helps in making plots
-        
-        client = FlowerClient(
-            cid, client_model, trainloader, valloader, self.diff_priv
-        )
-        self.flower_clients.append(client)
-        return client
-
-    def evaluate(
-        self,
-        server_round: int,
-        parameters: fl.common.NDArrays,
-        config: Dict[str, fl.common.Scalar],
-    ) -> Optional[Tuple[float, Dict[str, fl.common.Scalar]]]:
-        """
-        Evaluate the global model on the validation dataset and update the accuracies and losses.
-
-        Args:
-            server_round (int): The current round of the federated learning process.
-            parameters (fl.common.NDArrays): The global model parameters.
-            config (Dict[str, fl.common.Scalar]): Configuration dictionary.
-
-        Returns:
-            Optional[Tuple[float, Dict[str, fl.common.Scalar]]]: The evaluation loss and accuracy.
-        """
-        testloader = self.fed_dataset.valloaders[0]
-        
-        self.global_model.set_parameters(
-            parameters
-        )  # Update model with the latest parameters
-        loss, accuracy ,auc = self.global_model.evaluate(testloader, self.device)
-        self.auc.append(auc)
-        self.losses.append(loss)
-        self.accuracies.append(accuracy)
-        
-        return loss, {"accuracy": accuracy}
-
-    def run(self) -> None:
-        """
-        Run the federated learning process using Flower simulation.
-
-        Returns:
-            History: The history of the accuracies and losses during the training of each node 
-        """
-         # Increase the object store memory to the minimum allowed value or higher
-        ray_init_args = {"include_dashboard": False
-                         , "object_store_memory": 78643200
-                        }
-        self.fed_dataset.eng = None
-        
-        history = fl.simulation.start_simulation(
-            client_fn=self.client_fn,
-            num_clients=self.num_clients,
-            config=fl.server.ServerConfig(self.num_rounds),
-            strategy=self.strategy.strategy_object,
-            ray_init_args=ray_init_args,
-            client_resources = self.client_resources
-        )
-
-        return history
-

Medfl/LearningManager/strategy.py

@@ -1,82 +0,0 @@
-
-from collections import OrderedDict
-from typing import Dict, List, Optional, Tuple
-
-import flwr as fl
-import numpy as np
-
-import optuna
-
-
-
-
-class  Strategy:
-    """
-    A class representing a strategy for Federated Learning.
-
-    Attributes:
-        name (str): The name of the strategy. Default is "FedAvg".
-        fraction_fit (float): Fraction of clients to use for training during each round. Default is 1.0.
-        fraction_evaluate (float): Fraction of clients to use for evaluation during each round. Default is 1.0.
-        min_fit_clients (int): Minimum number of clients to use for training during each round. Default is 2.
-        min_evaluate_clients (int): Minimum number of clients to use for evaluation during each round. Default is 2.
-        min_available_clients (int): Minimum number of available clients required to start a round. Default is 2.
-        initial_parameters (Optional[]): The initial parameters of the server model 
-    Methods:
-     
-    """
-
-    def __init__(
-        self,
-        name: str = "FedAvg",
-        fraction_fit: float = 1.0,
-        fraction_evaluate: float = 1.0,
-        min_fit_clients: int = 2,
-        min_evaluate_clients: int = 2,
-        min_available_clients: int = 2,
-        initial_parameters = [],
-        evaluation_methode = "centralized"
-    ) -> None:
-        """
-        Initialize a Strategy object with the specified parameters.
-
-        Args:
-            name (str): The name of the strategy. Default is "FedAvg".
-            fraction_fit (float): Fraction of clients to use for training during each round. Default is 1.0.
-            fraction_evaluate (float): Fraction of clients to use for evaluation during each round. Default is 1.0.
-            min_fit_clients (int): Minimum number of clients to use for training during each round. Default is 2.
-            min_evaluate_clients (int): Minimum number of clients to use for evaluation during each round. Default is 2.
-            min_available_clients (int): Minimum number of available clients required to start a round. Default is 2.
-            initial_parameters (Optional[]): The initial parametres of the server model 
-            evaluation_methode ( "centralized" | "distributed")
-        """
-        self.fraction_fit = fraction_fit
-        self.fraction_evaluate = fraction_evaluate
-        self.min_fit_clients = min_fit_clients
-        self.min_evaluate_clients = min_evaluate_clients
-        self.min_available_clients = min_available_clients
-        self.initial_parameters = initial_parameters
-        self.evaluate_fn = None
-        self.name = name
-    
-    def optuna_fed_optimization(self, direction:str , hpo_rate:int , params_config):
-        self.study = optuna.create_study(direction=direction)
-        self.hpo_rate = hpo_rate
-        self.params_config = params_config
-    
- 
-    def create_strategy(self):
-        self.strategy_object = self.get_strategy_by_name()(
-            fraction_fit=self.fraction_fit,
-            fraction_evaluate=self.fraction_evaluate,
-            min_fit_clients=self.min_fit_clients,
-            min_evaluate_clients=self.min_evaluate_clients,
-            min_available_clients=self.min_available_clients,
-            initial_parameters=fl.common.ndarrays_to_parameters(self.initial_parameters),
-            evaluate_fn=self.evaluate_fn
-        )
-    def get_strategy_by_name(self):
-        return eval(f"fl.server.strategy.{self.name}")
-    
-    
-

Medfl/LearningManager/utils.py

@@ -1,331 +0,0 @@
-#!/usr/bin/env python3
-
-import pkg_resources
-import torch
-import yaml
-from sklearn.metrics import *
-from yaml.loader import SafeLoader
-
-
-from MEDfl.NetManager.database_connector import DatabaseManager
-
-# from scripts.base import *
-import json
-
-
-import pandas as pd
-import numpy as np
-
-import os
-import configparser
-
-import subprocess
-import ast
-
-from sqlalchemy import text
-
-
-# Get the directory of the current script
-current_directory = os.path.dirname(os.path.abspath(__file__))
-
-# Load configuration from the config file
-yaml_path = os.path.join(current_directory, 'params.yaml')
-
-with open(yaml_path) as g:
-    params = yaml.load(g, Loader=SafeLoader)
-
-# global_yaml_path = pkg_resources.resource_filename(__name__, "../../global_params.yaml")
-# with open(global_yaml_path) as g:
-#     global_params = yaml.load(g, Loader=SafeLoader)
-
-
-# Default path for the config file
-DEFAULT_CONFIG_PATH = 'db_config.ini'
-
-
-def load_db_config_dep():
-    config = os.environ.get('MEDfl_DB_CONFIG')
-
-    if config:
-        return ast.literal_eval(config)
-    else:
-        raise ValueError(f"MEDfl db config not found")
-
-# Function to allow users to set config path programmatically
-
-
-def set_db_config_dep(config_path):
-    config = configparser.ConfigParser()
-    config.read(config_path)
-    if (config['sqllite']):
-        os.environ['MEDfl_DB_CONFIG'] = str(dict(config['sqllite']))
-    else:
-        raise ValueError(f"mysql key not found in file '{config_path}'")
-
-
-
-def load_db_config():
-    """Read a dictionary from an environment variable."""
-    obj_str = os.getenv("MEDfl_DB_CONFIG")
-    if obj_str is not None:
-        return ast.literal_eval(obj_str)
-    else:
-        raise ValueError(f"Environment variable MEDfl_DB_CONFIG not found")
-
-# Function to allow users to set config path programmatically
-
-
-def set_db_config(config_path):
-    obj = {"database" : config_path}
-
-    """Store a dictionary as a string in an environment variable."""
-    obj_str = str(obj)
-    os.environ['MEDfl_DB_CONFIG'] = obj_str
-
-    
-
-
-
-
-# Create databas
-
-
-def create_MEDfl_db():
-    script_path = os.path.join(os.path.dirname(
-        __file__), 'scripts', 'create_db.sh')
-    subprocess.run(['sh', script_path], check=True)
-
-
-def custom_classification_report(y_true, y_pred_prob):
-    """
-    Compute custom classification report metrics including accuracy, sensitivity, specificity, precision, NPV,
-    F1-score, false positive rate, and true positive rate.
-
-    Args:
-        y_true (array-like): True labels.
-        y_pred (array-like): Predicted labels.
-
-    Returns:
-        dict: A dictionary containing custom classification report metrics.
-    """
-    y_pred = (y_pred_prob).round(
-    )  # Round absolute values of predicted probabilities to the nearest integer
-
-    auc = roc_auc_score(y_true, y_pred_prob)  # Calculate AUC
-
-    tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
-
-    # Accuracy
-    denominator_acc = tp + tn + fp + fn
-    acc = (tp + tn) / denominator_acc if denominator_acc != 0 else 0.0
-
-    # Sensitivity/Recall
-    denominator_sen = tp + fn
-    sen = tp / denominator_sen if denominator_sen != 0 else 0.0
-
-    # Specificity
-    denominator_sp = tn + fp
-    sp = tn / denominator_sp if denominator_sp != 0 else 0.0
-
-    # PPV/Precision
-    denominator_ppv = tp + fp
-    ppv = tp / denominator_ppv if denominator_ppv != 0 else 0.0
-
-#     NPV
-    denominator_npv = tn + fn
-    npv = tn / denominator_npv if denominator_npv != 0 else 0.0
-
-    # F1 Score
-    denominator_f1 = sen + ppv
-    f1 = 2 * (sen * ppv) / denominator_f1 if denominator_f1 != 0 else 0.0
-
-    # False Positive Rate
-    denominator_fpr = fp + tn
-    fpr = fp / denominator_fpr if denominator_fpr != 0 else 0.0
-
-    # True Positive Rate
-    denominator_tpr = tp + fn
-    tpr = tp / denominator_tpr if denominator_tpr != 0 else 0.0
-
-    return {
-        "confusion matrix": {"TP": tp, "FP": fp, "FN": fn, "TN": tn},
-        "Accuracy": round(acc, 3),
-        "Sensitivity/Recall": round(sen, 3),
-        "Specificity": round(sp, 3),
-        "PPV/Precision": round(ppv, 3),
-        "NPV": round(npv, 3),
-        "F1-score": round(f1, 3),
-        "False positive rate": round(fpr, 3),
-        "True positive rate": round(tpr, 3),
-        "auc": auc
-    }
-
-
-def test(model, test_loader, device=torch.device("cpu")):
-    """
-    Evaluate a model using a test loader and return a custom classification report.
-
-    Args:
-        model (torch.nn.Module): PyTorch model to evaluate.
-        test_loader (torch.utils.data.DataLoader): DataLoader for the test dataset.
-        device (torch.device, optional): Device for model evaluation. Default is "cpu".
-
-    Returns:
-        dict: A dictionary containing custom classification report metrics.
-    """
-
-    model.eval()
-    with torch.no_grad():
-        X_test, y_test = test_loader.dataset[:][0].to(
-            device), test_loader.dataset[:][1].to(device)
-        y_hat_prob = torch.squeeze(model(X_test), 1).cpu()
-
-    return custom_classification_report(y_test.cpu().numpy(), y_hat_prob.cpu().numpy())
-
-
-column_map = {"object": "VARCHAR(255)", "int64": "INT", "float64": "FLOAT"}
-
-
-def empty_db():
-    """
-    Empty the database by deleting records from multiple tables and resetting auto-increment counters.
-
-    Returns:
-        None
-    """
-    db_manager = DatabaseManager()
-    db_manager.connect()
-    my_eng = db_manager.get_connection()
-
-    # my_eng.execute(text(f"DELETE FROM  {'DataSets'}"))
-    my_eng.execute(text(f"DELETE FROM {'Nodes'}"))
-    my_eng.execute(text(f"DELETE FROM {'FedDatasets'}"))
-    my_eng.execute(text(f"DELETE FROM {'Networks'}"))
-    my_eng.execute(text(f"DELETE FROM {'FLsetup'}"))
-
-    my_eng.execute(text(f"DELETE FROM {'FLpipeline'}"))
-    my_eng.execute(text(f"ALTER TABLE {'Nodes'} AUTO_INCREMENT = 1"))
-    my_eng.execute(text(f"ALTER TABLE {'Networks'} AUTO_INCREMENT = 1"))
-    my_eng.execute(text(f"ALTER TABLE {'FedDatasets'} AUTO_INCREMENT = 1"))
-    my_eng.execute(text(f"ALTER TABLE {'FLsetup'} AUTO_INCREMENT = 1"))
-    my_eng.execute(text(f"ALTER TABLE {'FLpipeline'} AUTO_INCREMENT = 1"))
-    my_eng.execute(text(f"DELETE FROM {'testResults'}"))
-    my_eng.execute(text(f"DROP TABLE IF EXISTS {'MasterDataset'}"))
-    my_eng.execute(text(f"DROP TABLE IF EXISTS {'DataSets'}"))
-
-
-def get_pipeline_from_name(name):
-    """
-    Get the pipeline ID from its name in the database.
-
-    Args:
-        name (str): Name of the pipeline.
-
-    Returns:
-        int: ID of the pipeline.
-    """
-    db_manager = DatabaseManager()
-    db_manager.connect()
-    my_eng = db_manager.get_connection()
-
-    NodeId = int(
-        pd.read_sql(
-            text(f"SELECT id FROM FLpipeline WHERE name = '{name}'"), my_eng
-        ).iloc[0, 0]
-    )
-    return NodeId
-
-
-def get_pipeline_confusion_matrix(pipeline_id):
-    """
-    Get the global confusion matrix for a pipeline based on test results.
-
-    Args:
-        pipeline_id (int): ID of the pipeline.
-
-    Returns:
-        dict: A dictionary representing the global confusion matrix.
-    """
-    db_manager = DatabaseManager()
-    db_manager.connect()
-    my_eng = db_manager.get_connection()
-
-    data = pd.read_sql(
-        text(
-            f"SELECT confusionmatrix FROM testResults WHERE pipelineid = '{pipeline_id}'"), my_eng
-    )
-
-    # Convert the column of strings into a list of dictionaries representing confusion matrices
-    confusion_matrices = [
-        json.loads(matrix.replace("'", "\"")) for matrix in data['confusionmatrix']
-    ]
-
-    # Initialize variables for global confusion matrix
-    global_TP = global_FP = global_FN = global_TN = 0
-
-    # Iterate through each dictionary and sum the corresponding values for each category
-    for matrix in confusion_matrices:
-        global_TP += matrix['TP']
-        global_FP += matrix['FP']
-        global_FN += matrix['FN']
-        global_TN += matrix['TN']
-
-    # Create a global confusion matrix as a dictionary
-    global_confusion_matrix = {
-        'TP': global_TP,
-        'FP': global_FP,
-        'FN': global_FN,
-        'TN': global_TN
-    }
-    # Return the list of dictionaries representing confusion matrices
-    return global_confusion_matrix
-
-
-def get_node_confusion_matrix(pipeline_id, node_name):
-    """
-    Get the confusion matrix for a specific node in a pipeline based on test results.
-
-    Args:
-        pipeline_id (int): ID of the pipeline.
-        node_name (str): Name of the node.
-
-    Returns:
-        dict: A dictionary representing the confusion matrix for the specified node.
-    """
-    db_manager = DatabaseManager()
-    db_manager.connect()
-    my_eng = db_manager.get_connection()
-
-    data = pd.read_sql(
-        text(
-            f"SELECT confusionmatrix FROM testResults WHERE pipelineid = '{pipeline_id}' AND nodename = '{node_name}'"), my_eng
-    )
-
-    # Convert the column of strings into a list of dictionaries representing confusion matrices
-    confusion_matrices = [
-        json.loads(matrix.replace("'", "\"")) for matrix in data['confusionmatrix']
-    ]
-
-    # Return the list of dictionaries representing confusion matrices
-    return confusion_matrices[0]
-
-
-def get_pipeline_result(pipeline_id):
-    """
-    Get the test results for a pipeline.
-
-    Args:
-        pipeline_id (int): ID of the pipeline.
-
-    Returns:
-        pandas.DataFrame: DataFrame containing test results for the specified pipeline.
-    """
-    db_manager = DatabaseManager()
-    db_manager.connect()
-    my_eng = db_manager.get_connection()
-
-    data = pd.read_sql(
-        text(
-            f"SELECT * FROM testResults WHERE pipelineid = '{pipeline_id}'"), my_eng
-    )
-    return data

Medfl/NetManager/__init__.py

@@ -1,10 +0,0 @@
-# # MEDfl/NetworkManager/__init__.py
-
-# # Import modules from this package
-# from .dataset import *
-# from .flsetup import *
-# from .net_helper import *
-# from .net_manager_queries import *
-# from .network import *
-# from .node import *
-# from .database_connector import *

Medfl/NetManager/database_connector.py

@@ -1,43 +0,0 @@
-import os
-import subprocess
-from sqlalchemy import create_engine
-from configparser import ConfigParser
-
-class DatabaseManager:
-    def __init__(self):
-        from MEDfl.LearningManager.utils import load_db_config
-        db_config = load_db_config()
-        if db_config:
-            self.config = db_config
-        else:
-            self.config = None
-        self.engine = None
-
-    def connect(self):
-        if not self.config:
-            raise ValueError("Database configuration not loaded. Use load_db_config() or set_config_path() first.")
-        # Assuming the SQLite database file path is provided in the config with the key 'database'
-        database_path = self.config['database']
-        connection_string = f"sqlite:///{database_path}"
-        self.engine = create_engine(connection_string, pool_pre_ping=True)
-
-    def get_connection(self):
-        if not self.engine:
-            self.connect()
-        return self.engine.connect()
-
-    def create_MEDfl_db(self, path_to_csv):
-        # Get the directory of the current script
-        current_directory = os.path.dirname(__file__)
-
-        # Define the path to the create_db.py script
-        create_db_script_path = os.path.join(current_directory, '..', 'scripts', 'create_db.py')
-
-        # Execute the create_db.py script
-        subprocess.run(['python', create_db_script_path, path_to_csv], check=True)
-
-        return
-
-    def close(self):
-        if self.engine:
-            self.engine.dispose()