MEDfl 2.0.4.dev1__py3-none-any.whl → 2.0.4.dev3__py3-none-any.whl

Medfl/NetManager/dataset.py

@@ -1,92 +0,0 @@
-import pandas as pd
-from sqlalchemy import text
-
-from .net_helper import *
-from .net_manager_queries import (DELETE_DATASET, INSERT_DATASET,
-                                  SELECT_ALL_DATASET_NAMES)
-from MEDfl.NetManager.database_connector import DatabaseManager
-
-class DataSet:
-    def __init__(self, name: str, path: str, engine=None):
-        """
-        Initialize a DataSet object.
-
-        :param name: The name of the dataset.
-        :type name: str
-        :param path: The file path of the dataset CSV file.
-        :type path: str
-        """
-        self.name = name
-        self.path = path
-        db_manager = DatabaseManager()
-        db_manager.connect()
-        self.engine = db_manager.get_connection()
-
-    def validate(self):
-        """
-        Validate name and path attributes.
-
-        :raises TypeError: If name or path is not a string.
-        """
-        if not isinstance(self.name, str):
-            raise TypeError("name argument must be a string")
-
-        if not isinstance(self.path, str):
-            raise TypeError("path argument must be a string")
-
-    def upload_dataset(self, NodeId=-1):
-        """
-        Upload the dataset to the database.
-
-        :param NodeId: The NodeId associated with the dataset.
-        :type NodeId: int
-
-        Notes:
-        - Assumes the file at self.path is a valid CSV file.
-        - The dataset is uploaded to the 'DataSets' table in the database.
-        """
-
-        data_df = pd.read_csv(self.path)
-        nodeId = NodeId
-        columns = data_df.columns.tolist()
-        
-
-        data_df = process_eicu(data_df)
-        for index, row in data_df.iterrows():
-            query_1 = "INSERT INTO DataSets(DataSetName,nodeId," + "".join(
-                f"{x}," for x in columns
-            )
-            query_2 = f" VALUES ('{self.name}',{nodeId}, " + "".join(
-                f"{is_str(data_df, row, x)}," for x in columns
-            )
-            query = query_1[:-1] + ")" + query_2[:-1] + ")"
-             
-            self.engine.execute(text(query))
-
-    def delete_dataset(self):
-        """
-        Delete the dataset from the database.
-
-        Notes:
-        - Assumes the dataset name is unique in the 'DataSets' table.
-        """
-        self.engine.execute(text(DELETE_DATASET), {"name": self.name})
-
-    def update_data(self):
-        """
-        Update the data in the dataset.
-
-        Not implemented yet.
-        """
-        pass
-
-    @staticmethod
-    def list_alldatasets(engine):
-        """
-        List all dataset names from the 'DataSets' table.
-
-        :returns: A DataFrame containing the names of all datasets in the 'DataSets' table.
-        :rtype: pd.DataFrame
-        """
-        res = pd.read_sql(text(SELECT_ALL_DATASET_NAMES), engine)
-        return res

Medfl/NetManager/flsetup.py

@@ -1,320 +0,0 @@
-from datetime import datetime
-
-
-from torch.utils.data import random_split, DataLoader, Dataset
-
-from MEDfl.LearningManager.federated_dataset import FederatedDataset
-from .net_helper import *
-from .net_manager_queries import *  # Import the sql_queries module
-from .network import Network
-
-from .node import Node
-
-from MEDfl.NetManager.database_connector import DatabaseManager
-
-
-class FLsetup:
-    def __init__(self, name: str, description: str, network: Network):
-        """Initialize a Federated Learning (FL) setup.
-
-        Args:
-            name (str): The name of the FL setup.
-            description (str): A description of the FL setup.
-            network (Network): An instance of the Network class representing the network architecture.
-        """
-        self.name = name
-        self.description = description
-        self.network = network
-        self.column_name = None
-        self.auto = 1 if self.column_name is not None else 0
-        self.validate()
-        self.fed_dataset = None
-
-        db_manager = DatabaseManager()
-        db_manager.connect()
-        self.eng = db_manager.get_connection()
-
-        
-
-    def validate(self):
-        """Validate name, description, and network."""
-        if not isinstance(self.name, str):
-            raise TypeError("name argument must be a string")
-
-        if not isinstance(self.description, str):
-            raise TypeError("description argument must be a string")
-
-        if not isinstance(self.network, Network):
-            raise TypeError(
-                "network argument must be a MEDfl.NetManager.Network "
-            )
-
-    def create(self):
-        """Create an FL setup."""
-        creation_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        netid = get_netid_from_name(self.network.name)
-        self.eng.execute(
-            text(CREATE_FLSETUP_QUERY),
-            {
-                "name": self.name,
-                "description": self.description,
-                "creation_date": creation_date,
-                "net_id": netid,
-                "column_name": self.column_name,
-            },
-        )
-        self.id = get_flsetupid_from_name(self.name)
-
-    def delete(self):
-        """Delete the FL setup."""
-        if self.fed_dataset is not None:
-            self.fed_dataset.delete_Flsetup(FLsetupId=self.id)
-        self.eng.execute(text(DELETE_FLSETUP_QUERY), {"name": self.name})
-
-    @classmethod
-    def read_setup(cls, FLsetupId: int):
-        """Read the FL setup by FLsetupId.
-
-        Args:
-            FLsetupId (int): The id of the FL setup to read.
-
-        Returns:
-            FLsetup: An instance of the FLsetup class with the specified FLsetupId.
-        """
-        db_manager = DatabaseManager()
-        db_manager.connect()
-        my_eng = db_manager.get_connection()
-
-        res = pd.read_sql(
-            text(READ_SETUP_QUERY), my_eng, params={"flsetup_id": FLsetupId}
-        ).iloc[0]
-        
-        network_res = pd.read_sql(
-            text(READ_NETWORK_BY_ID_QUERY),
-            my_eng,
-            params={"net_id": int(res["NetId"])},
-        ).iloc[0]
-        network = Network(network_res["NetName"])
-        setattr(network, "id", res["NetId"])
-        fl_setup = cls(res["name"], res["description"], network)
-        if res["column_name"] == str(None):
-            res["column_name"] = None
-        setattr(fl_setup, "column_name", res["column_name"])
-        setattr(fl_setup, "id", res["FLsetupId"])
-
-        return fl_setup
-
-    @staticmethod
-    def list_allsetups():
-        """List all the FL setups.
-
-        Returns:
-            DataFrame: A DataFrame containing information about all the FL setups.
-        """
-        db_manager = DatabaseManager()
-        db_manager.connect()
-        my_eng = db_manager.get_connection()
-
-        Flsetups = pd.read_sql(text(READ_ALL_SETUPS_QUERY), my_eng)
-        return Flsetups
-
-    def create_nodes_from_master_dataset(self, params_dict: dict):
-        """Create nodes from the master dataset.
-
-        Args:
-            params_dict (dict): A dictionary containing parameters for node creation.
-                - column_name (str): The name of the column in the MasterDataset used to create nodes.
-                - train_nodes (list): A list of node names that will be used for training.
-                - test_nodes (list): A list of node names that will be used for testing.
-
-        Returns:
-            list: A list of Node instances created from the master dataset.
-        """
-        assert "column_name" in params_dict.keys()
-        column_name, train_nodes, test_nodes = (
-            params_dict["column_name"],
-            params_dict["train_nodes"],
-            params_dict["test_nodes"],
-        )
-        self.column_name = column_name
-        self.auto = 1
-
-        # Update the Column name of the auto flSetup
-        query = f"UPDATE FLsetup SET column_name = '{column_name}' WHERE name = '{self.name}'"
-        self.eng.execute(text(query))
-        
-
-        # Add Network to DB
-        # self.network.create_network()
-
-        netid = get_netid_from_name(self.network.name)
-
-        assert self.network.mtable_exists == 1
-        node_names = pd.read_sql(
-            text(READ_DISTINCT_NODES_QUERY.format(column_name)), self.eng
-        )
-
-        nodes = [Node(val[0], 1) for val in node_names.values.tolist()]
-
-        used_nodes = []
-
-        for node in nodes:
-            if node.name in train_nodes:
-                node.train = 1
-                node.create_node(netid)
-                used_nodes.append(node)
-            if node.name in test_nodes:
-                node.train =0
-                node.create_node(netid) 
-                used_nodes.append(node)
-        return used_nodes
-
-    def create_dataloader_from_node(
-        self,
-        node: Node,
-        output,
-        fill_strategy="mean",  fit_encode=[], to_drop=[],
-        train_batch_size: int = 32,
-        test_batch_size: int = 1,
-        split_frac: float = 0.2,
-        dataset: Dataset = None,
-
-    ):
-        """Create DataLoader from a Node.
-
-        Args:
-            node (Node): The node from which to create DataLoader.
-            train_batch_size (int): The batch size for training data.
-            test_batch_size (int): The batch size for test data.
-            split_frac (float): The fraction of data to be used for training.
-            dataset (Dataset): The dataset to use. If None, the method will read the dataset from the node.
-
-        Returns:
-            DataLoader: The DataLoader instances for training and testing.
-        """
-        if dataset is None:
-            if self.column_name is not None:
-                dataset = process_data_after_reading(
-                    node.get_dataset(self.column_name), output, fill_strategy=fill_strategy, fit_encode=fit_encode, to_drop=to_drop
-                )
-            else:
-                dataset = process_data_after_reading(
-                    node.get_dataset(), output, fill_strategy=fill_strategy, fit_encode=fit_encode, to_drop=to_drop)
-
-        dataset_size = len(dataset)
-        traindata_size = int(dataset_size * (1 - split_frac))
-        traindata, testdata = random_split(
-            dataset, [traindata_size, dataset_size - traindata_size]
-        )
-        trainloader, testloader = DataLoader(
-            traindata, batch_size=train_batch_size
-        ), DataLoader(testdata, batch_size=test_batch_size)
-        return trainloader, testloader
-
-    def create_federated_dataset(
-        self, output, fill_strategy="mean",  fit_encode=[], to_drop=[], val_frac=0.1, test_frac=0.2
-    ) -> FederatedDataset:
-        """Create a federated dataset.
-
-        Args:
-            output (string): the output feature of the dataset
-            val_frac (float): The fraction of data to be used for validation.
-            test_frac (float): The fraction of data to be used for testing.
-
-        Returns:
-            FederatedDataset: The FederatedDataset instance containing train, validation, and test data.
-        """
-        
-        if not self.column_name:
-            to_drop.extend(["DataSetName" , "NodeId" , "DataSetId"])
-        else :
-            to_drop.extend(["PatientId"]) 
-            
-        netid = self.network.id
-        train_nodes = pd.read_sql(
-            text(
-                f"SELECT Nodes.NodeName  FROM Nodes WHERE Nodes.NetId = {netid} AND Nodes.train = 1 "
-            ),
-            self.eng,
-        )
-        test_nodes = pd.read_sql(
-            text(
-                f"SELECT Nodes.NodeName  FROM Nodes WHERE Nodes.NetId = {netid} AND Nodes.train = 0 "
-            ),
-            self.eng,
-        )
-
-        train_nodes = [
-            Node(val[0], 1, test_frac) for val in train_nodes.values.tolist()
-        ]
-        test_nodes = [Node(val[0], 0) for val in test_nodes.values.tolist()]
-
-        trainloaders, valloaders, testloaders = [], [], []
-        # if len(test_nodes) == 0:
-        #     raise "test node empty"
-        if test_nodes is None:
-            _, testloader = self.create_dataloader_from_node(
-                train_nodes[0], output, fill_strategy=fill_strategy, fit_encode=fit_encode, to_drop=to_drop)
-            testloaders.append(testloader)
-        else:
-            for train_node in train_nodes:
-                train_valloader, testloader = self.create_dataloader_from_node(
-                    train_node, output, fill_strategy=fill_strategy,
-                    fit_encode=fit_encode, to_drop=to_drop,)
-                trainloader, valloader = self.create_dataloader_from_node(
-                    train_node,
-                    output, fill_strategy=fill_strategy, fit_encode=fit_encode, to_drop=to_drop,
-                    test_batch_size=32,
-                    split_frac=val_frac,
-                    dataset=train_valloader.dataset,
-                )
-                trainloaders.append(trainloader)
-                valloaders.append(valloader)
-                testloaders.append(testloader)
-
-            for test_node in test_nodes:
-                _, testloader = self.create_dataloader_from_node(
-                    test_node, output, fill_strategy=fill_strategy, fit_encode=fit_encode, to_drop=to_drop, split_frac=1.0
-                )
-                testloaders.append(testloader)
-        train_nodes_names = [node.name for node in train_nodes]
-        test_nodes_names = train_nodes_names + [
-            node.name for node in test_nodes
-        ]
-        
-        # test_nodes_names = [
-        #     node.name for node in test_nodes
-        # ]
-
-        # Add FlSetup on to the DataBase
-        # self.create()
-
-        # self.network.update_network(FLsetupId=self.id)
-        fed_dataset = FederatedDataset(
-            self.name + "_Feddataset",
-            train_nodes_names,
-            test_nodes_names,
-            trainloaders,
-            valloaders,
-            testloaders,
-        )
-        self.fed_dataset = fed_dataset
-        self.fed_dataset.create(self.id)
-        return self.fed_dataset
-    
-    
-
-
-    def get_flDataSet(self):
-        """
-        Retrieve the federated dataset associated with the FL setup using the FL setup's name.
- 
-        Returns:
-            pandas.DataFrame: DataFrame containing the federated dataset information.
-        """
-        return pd.read_sql(
-            text(
-                f"SELECT * FROM FedDatasets WHERE FLsetupId = {get_flsetupid_from_name(self.name)}"
-            ),
-            self.eng,
-        )

Medfl/NetManager/net_helper.py

@@ -1,254 +0,0 @@
-from sklearn.preprocessing import LabelEncoder
-from sklearn.impute import SimpleImputer
-
-from sqlalchemy import text
-
-import torch
-import pandas as pd
-from torch.utils.data import TensorDataset
-import numpy as np
-
-from MEDfl.NetManager.database_connector import DatabaseManager
-
-
-def is_str(data_df, row, x):
-    """
-    Check if a column in a DataFrame is of type 'object' and convert the value accordingly.
-
-    Args:
-        data_df (pandas.DataFrame): DataFrame containing the data.
-        row (pandas.Series): Data row.
-        x (str): Column name.
-
-    Returns:
-        str or float: Processed value based on the column type.
-    """
-    if data_df[x].dtype == "object":
-        x = f"'{row[x]}'"
-    else:
-        x = row[x]
-    return x
-
-
-def process_eicu(data_df):
-    """
-    Process eICU data by filling missing values with mean and replacing NaNs with 'Unknown'.
-
-    Args:
-        data_df (pandas.DataFrame): Input data.
-
-    Returns:
-        pandas.DataFrame: Processed data.
-    """
-    # Identify numeric and non-numeric columns
-    numeric_columns = data_df.select_dtypes(include=[np.number]).columns
-    non_numeric_columns = data_df.select_dtypes(exclude=[np.number]).columns
-
-    # Fill NaN in numeric columns with mean
-    data_df[numeric_columns] = data_df[numeric_columns].fillna(
-        data_df[numeric_columns].mean())
-
-    # Fill NaN in non-numeric columns with 'Unknown'
-    data_df[non_numeric_columns] = data_df[non_numeric_columns].fillna(
-        'Unknown')
-
-    try:
-        data_df = data_df.reset_index(drop=True)
-    except:
-        pass
-
-    return data_df
-
-
-# remove indiserd columns after reading from the DB
-def process_data_after_reading(data, output, fill_strategy="mean",  fit_encode=[], to_drop=[]):
-    """
-    Process data after reading from the database, including encoding, dropping columns, and creating a PyTorch TensorDataset.
-
-    Args:
-        data (pandas.DataFrame): Input data.
-        output (str): Output column name.
-        fill_strategy (str, optional): Imputation strategy for missing values. Default is "mean".
-        fit_encode (list, optional): List of columns to be label-encoded. Default is an empty list.
-        to_drop (list, optional): List of columns to be dropped from the DataFrame. Default is an empty list.
-
-    Returns:
-        torch.utils.data.TensorDataset: Processed data as a PyTorch TensorDataset.
-    """
-
-    # Check if there is a DataSet assigned to the node
-    if (len(data) == 0):
-        raise "Node doesn't Have dataSet"
-
-    encoder = LabelEncoder()
-    # En Code some columns
-    for s in fit_encode:
-        try:
-            data[s] = encoder.fit_transform(data[s])
-        except:
-            raise print(s)
-
-    # The output of the DATA
-    y = data[output]
-
-    X = data
-
-    # remove indisered columns when reading the dataframe from the DB
-    for column in to_drop:
-        try:
-            X = X.drop(
-                [column], axis=1
-            )
-        except Exception as e:
-            raise e
-
-    # Get the DATAset Features
-    features = [col for col in X.columns if col != output]
-
-    # Impute missing values using the mean strategy
-    try:
-        imputer = SimpleImputer(strategy=fill_strategy)
-        X[features] = imputer.fit_transform(X[features])
-    except:
-        print()
-
-    X = torch.tensor(X.values, dtype=torch.float32)
-    y = torch.tensor(y.values, dtype=torch.float32)
-    data = TensorDataset(X, y)
-
-    return data
-
-
-def get_nodeid_from_name(name):
-    """
-    Get the NodeId from the Nodes table based on the NodeName.
-
-    Args:
-        name (str): Node name.
-
-    Returns:
-        int or None: NodeId or None if not found.
-    """
-    db_manager = DatabaseManager()
-    db_manager.connect()
-    my_eng = db_manager.get_connection()
-
-    result_proxy = my_eng.execute(f"SELECT NodeId FROM Nodes WHERE NodeName = '{name}'")
-    NodeId = int(result_proxy.fetchone()[0])
-    return NodeId
-
-
-def get_netid_from_name(name):
-    """
-    Get the Network Id from the Networks table based on the NetName.
-
-    Args:
-        name (str): Network name.
-
-    Returns:
-        int or None: NetId or None if not found.
-    """
-    db_manager = DatabaseManager()
-    db_manager.connect()
-    my_eng = db_manager.get_connection()
-
-    try:
-        result_proxy = my_eng.execute(f"SELECT NetId FROM Networks WHERE NetName = '{name}'")
-        NetId = int(result_proxy.fetchone()[0])
-    except:
-        NetId = None
-    return NetId
-
-
-def get_flsetupid_from_name(name):
-    """
-    Get the FLsetupId from the FLsetup table based on the FL setup name.
-
-    Args:
-        name (str): FL setup name.
-
-    Returns:
-        int or None: FLsetupId or None if not found.
-    """
-    db_manager = DatabaseManager()
-    db_manager.connect()
-    my_eng = db_manager.get_connection()
-
-    try:
-        
-         result_proxy = my_eng.execute(f"SELECT FLsetupId FROM FLsetup WHERE name = '{name}'")
-         id = int(result_proxy.fetchone()[0])
-   
-    except:
-        id = None
-    return id
-
-
-def get_flpipeline_from_name(name):
-    """
-    Get the FLpipeline Id from the FLpipeline table based on the FL pipeline name.
-
-    Args:
-        name (str): FL pipeline name.
-
-    Returns:
-        int or None: FLpipelineId or None if not found.
-    """
-    db_manager = DatabaseManager()
-    db_manager.connect()
-    my_eng = db_manager.get_connection()
-
-    try:
-
-        result_proxy = my_eng.execute(f"SELECT id FROM FLpipeline WHERE name = '{name}'")
-        id = int(result_proxy.fetchone()[0])
-    except:
-        id = None
-    return id
-
-
-def get_feddataset_id_from_name(name):
-    """
-    Get the Federated dataset Id from the FedDatasets table based on the federated dataset name.
-
-    Args:
-        name (str): Federated dataset name.
-
-    Returns:
-        int or None: FedId or None if not found.
-    """
-    db_manager = DatabaseManager()
-    db_manager.connect()
-    my_eng = db_manager.get_connection()
-
-    try:
-        
-        result_proxy = my_eng.execute(f"SELECT FedId FROM FedDatasets WHERE name = '{name}'")
-        id = int(result_proxy.fetchone()[0])
-    except:
-        id = None
-    return id
-
-
-def master_table_exists():
-    """
-    Check if the MasterDataset table exists in the database.
-
-    Returns:
-        bool: True if the table exists, False otherwise.
-    """
-    try:
-        db_manager = DatabaseManager()
-        db_manager.connect()
-        my_eng = db_manager.get_connection()
-
-        # SQLite-specific query to check if table exists
-        sql_query = text("SELECT name FROM sqlite_master WHERE type='table' AND name='MasterDataset'")
-        result = my_eng.execute(sql_query)
-        exists = result.fetchone() is not None
-        return exists
-
-    except Exception as e:
-        print(f"Error checking MasterDataset table existence: {e}")
-        return False
-