MEDfl 0.1.0__py3-none-any.whl → 0.1.4__py3-none-any.whl

Medfl/NetManager/net_helper.py

@@ -0,0 +1,243 @@
+from sklearn.preprocessing import LabelEncoder
+from sklearn.impute import SimpleImputer
+
+from sqlalchemy import text
+
+import torch
+import pandas as pd
+from torch.utils.data import TensorDataset
+import numpy as np
+
+
+from scripts.base import my_eng
+
+
+def is_str(data_df, row, x):
+    """
+    Check if a column in a DataFrame is of type 'object' and convert the value accordingly.
+
+    Args:
+        data_df (pandas.DataFrame): DataFrame containing the data.
+        row (pandas.Series): Data row.
+        x (str): Column name.
+
+    Returns:
+        str or float: Processed value based on the column type.
+    """
+    if data_df[x].dtype == "object":
+        x = f"'{row[x]}'"
+    else:
+        x = row[x]
+    return x
+
+
+def process_eicu(data_df):
+    """
+    Process eICU data by filling missing values with mean and replacing NaNs with 'Unknown'.
+
+    Args:
+        data_df (pandas.DataFrame): Input data.
+
+    Returns:
+        pandas.DataFrame: Processed data.
+    """
+    # Identify numeric and non-numeric columns
+    numeric_columns = data_df.select_dtypes(include=[np.number]).columns
+    non_numeric_columns = data_df.select_dtypes(exclude=[np.number]).columns
+
+    # Fill NaN in numeric columns with mean
+    data_df[numeric_columns] = data_df[numeric_columns].fillna(data_df[numeric_columns].mean())
+
+    # Fill NaN in non-numeric columns with 'Unknown'
+    data_df[non_numeric_columns] = data_df[non_numeric_columns].fillna('Unknown')
+
+    try:
+        data_df = data_df.reset_index(drop=True)
+    except:
+        pass
+
+    return data_df
+
+
+# remove indiserd columns after reading from the DB
+def process_data_after_reading(data, output, fill_strategy="mean",  fit_encode=[], to_drop=[]):
+    """
+    Process data after reading from the database, including encoding, dropping columns, and creating a PyTorch TensorDataset.
+
+    Args:
+        data (pandas.DataFrame): Input data.
+        output (str): Output column name.
+        fill_strategy (str, optional): Imputation strategy for missing values. Default is "mean".
+        fit_encode (list, optional): List of columns to be label-encoded. Default is an empty list.
+        to_drop (list, optional): List of columns to be dropped from the DataFrame. Default is an empty list.
+
+    Returns:
+        torch.utils.data.TensorDataset: Processed data as a PyTorch TensorDataset.
+    """
+
+    # Check if there is a DataSet assigned to the node
+    if (len(data) == 0):
+        raise "Node doesn't Have dataSet"
+
+    encoder = LabelEncoder()
+    # En Code some columns
+    for s in fit_encode:
+        try:
+            data[s] = encoder.fit_transform(data[s])
+        except:
+            raise print(s)
+
+    # The output of the DATA
+    y = data[output]
+
+    X = data
+
+    
+    # remove indisered columns when reading the dataframe from the DB
+    for column in to_drop:
+        try:
+            X = X.drop(
+                [column], axis=1
+            )
+        except Exception as e:
+            raise e
+
+    
+    # Get the DATAset Features
+    features = [col for col in X.columns if col != output]
+
+    # Impute missing values using the mean strategy
+    try:
+        imputer = SimpleImputer(strategy=fill_strategy)
+        X[features] = imputer.fit_transform(X[features])
+    except:
+        print()
+
+    X = torch.tensor(X.values, dtype=torch.float32)
+    y = torch.tensor(y.values, dtype=torch.float32)
+    data = TensorDataset(X, y)
+
+    return data
+
+
+def get_nodeid_from_name(name):
+    """
+    Get the NodeId from the Nodes table based on the NodeName.
+
+    Args:
+        name (str): Node name.
+
+    Returns:
+        int or None: NodeId or None if not found.
+    """
+
+    NodeId = int(
+        pd.read_sql(
+            text(f"SELECT NodeId FROM Nodes WHERE NodeName = '{name}'"), my_eng
+        ).iloc[0, 0]
+    )
+    return NodeId
+
+
+def get_netid_from_name(name):
+    """
+    Get the Network Id from the Networks table based on the NetName.
+
+    Args:
+        name (str): Network name.
+
+    Returns:
+        int or None: NetId or None if not found.
+    """
+    try:
+        NetId = int(
+            pd.read_sql(
+                text(f"SELECT NetId FROM Networks WHERE NetName = '{name}'"),
+                my_eng,
+            ).iloc[0, 0]
+        )
+    except:
+        NetId = None
+    return NetId
+
+
+def get_flsetupid_from_name(name):
+    """
+    Get the FLsetupId from the FLsetup table based on the FL setup name.
+
+    Args:
+        name (str): FL setup name.
+
+    Returns:
+        int or None: FLsetupId or None if not found.
+    """
+    try:
+        id = int(
+            pd.read_sql(
+                text(f"SELECT FLsetupId FROM FLsetup WHERE name = '{name}'"),
+                my_eng,
+            ).iloc[0, 0]
+        )
+    except:
+        id = None
+    return id
+
+
+def get_flpipeline_from_name(name):
+    """
+    Get the FLpipeline Id from the FLpipeline table based on the FL pipeline name.
+
+    Args:
+        name (str): FL pipeline name.
+
+    Returns:
+        int or None: FLpipelineId or None if not found.
+    """
+    try:
+        id = int(
+            pd.read_sql(
+                text(f"SELECT id FROM FLpipeline WHERE name = '{name}'"),
+                my_eng,
+            ).iloc[0, 0]
+        )
+    except:
+        id = None
+    return id
+
+
+def get_feddataset_id_from_name(name):
+    """
+    Get the Federated dataset Id from the FedDatasets table based on the federated dataset name.
+
+    Args:
+        name (str): Federated dataset name.
+
+    Returns:
+        int or None: FedId or None if not found.
+    """
+    try:
+        id = int(
+            pd.read_sql(
+                text(f"SELECT FedId FROM FedDatasets WHERE name = '{name}'"),
+                my_eng,
+            ).iloc[0, 0]
+        )
+    except:
+        id = None
+    return id
+
+
+def master_table_exists():
+    """
+    Check if the MasterDataset table exists in the database.
+
+    Returns:
+        bool: True if the table exists, False otherwise.
+    """
+    
+    return pd.read_sql(
+        text(
+            " SELECT EXISTS ( SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_NAME = 'MasterDataset' )"
+        ),
+        my_eng,
+    ).values[0][0]

Medfl/NetManager/net_manager_queries.py

@@ -0,0 +1,137 @@
+from .net_helper import is_str
+
+INSERT_DATASET = """
+    INSERT INTO DataSets(DataSetName, NodeId, {columns})
+    VALUES (:name, :NodeId, {values})
+"""
+DELETE_DATASET = """
+    DELETE FROM DataSets WHERE DataSetName = :name
+"""
+
+SELECT_ALL_DATASET_NAMES = """
+    SELECT DISTINCT DataSetName,NodeId FROM DataSets
+"""
+
+SELECT_DATASET_BY_NAME = """
+    SELECT * FROM DataSets WHERE DataSetName = :name
+"""
+
+# node queries
+# sql_queries.py
+
+INSERT_NODE_QUERY = (
+    "INSERT INTO Nodes(NodeName,NetId,train) VALUES ('{}',{}, {})"
+)
+DELETE_NODE_QUERY = "DELETE FROM Nodes WHERE NodeName = '{}'"
+SELECT_MASTER_COLUMNS_QUERY = "SELECT * FROM MasterDataset LIMIT 1"
+SELECT_DATASET_BY_COLUMN_QUERY = "SELECT * FROM MasterDataset WHERE {} = '{}'"
+SELECT_DATASET_BY_NODE_ID_QUERY = "SELECT * FROM DataSets WHERE NodeId = {}"
+
+SELECT_ALL_DATASETS_QUERY = "SELECT DISTINCT DataSetName,NodeName FROM DataSets,Nodes WHERE Nodes.NodeName = '{}' and Nodes.NodeId = DataSets.NodeId"
+SELECT_ALL_NODES_QUERY = "SELECT * FROM Nodes"
+
+
+# SQL query to insert a new network
+INSERT_NETWORK_QUERY = "INSERT INTO Networks(NetName) VALUES ('{name}')"
+
+# SQL query to delete a network
+DELETE_NETWORK_QUERY = "DELETE FROM Networks WHERE NetName = '{name}'"
+
+# SQL query to delete a network
+GET_NETWORK_QUERY = "SELECT * FROM Networks WHERE NetName = '{name}'"
+
+
+# SQL query to update a network
+UPDATE_NETWORK_QUERY = (
+    "UPDATE Networks SET FLsetupId = {FLsetupId} WHERE NetId = {id}"
+)
+
+# SQL query to retrieve all nodes for a network
+LIST_ALL_NODES_QUERY = "SELECT Nodes.NodeName, Networks.NetName FROM Nodes, Networks WHERE Networks.NetName = '{name}' AND Networks.NetId = Nodes.NetId"
+
+# SQL query to create the MasterDataset table
+CREATE_MASTER_DATASET_TABLE_QUERY = """
+CREATE TABLE IF NOT EXISTS MasterDataset (
+    PatientId INT NOT NULL AUTO_INCREMENT,
+    {},
+    PRIMARY KEY (PatientId)
+);
+"""
+
+# SQL query to create the datasets table
+CREATE_DATASETS_TABLE_QUERY = """
+CREATE TABLE IF NOT EXISTS Datasets (
+                     DataSetId INT NOT NULL AUTO_INCREMENT, 
+                     DataSetName VARCHAR(255), 
+                     NodeId INT,
+                     {},
+                     PRIMARY KEY (DataSetId)
+                     
+);
+"""
+
+# SQL query to insert dataset values
+INSERT_DATASET_VALUES_QUERY = "INSERT INTO MasterDataset({columns}, NodeId) VALUES ('{name}', {nodeId}, {values})"
+
+
+# FL setup_queries
+# sql_queries.py
+
+CREATE_FLSETUP_QUERY = """
+    INSERT INTO FLsetup (name, description, creation_date, NetId, column_name)
+    VALUES (:name, :description, :creation_date, :net_id, :column_name)
+"""
+
+DELETE_FLSETUP_QUERY = """
+    DELETE FROM FLsetup
+    WHERE name = :name
+"""
+
+UPDATE_FLSETUP_QUERY = UPDATE_NETWORK_QUERY = (
+    "UPDATE FLsetup SET column_name ='{column_name}' WHERE name ='{FLsetupName}'"
+)
+
+
+READ_SETUP_QUERY = """
+    SELECT * FROM FLsetup
+    WHERE FLsetupId = :flsetup_id
+"""
+
+READ_ALL_SETUPS_QUERY = """
+    SELECT * FROM FLsetup
+"""
+
+READ_NETWORK_BY_ID_QUERY = """
+    SELECT * FROM Networks
+    WHERE NetId = :net_id
+"""
+
+READ_DISTINCT_NODES_QUERY = """
+SELECT DISTINCT {} FROM MasterDataset 
+"""
+
+
+# FederatedDataset Queries
+INSERT_FLDATASET_QUERY = (
+    "INSERT INTO FedDatasets(name, FLsetupId) VALUES (:name, :FLsetupId)"
+)
+DELETE_FLDATASET_BY_SETUP_AND_PIPELINE_QUERY = "DELETE FROM FedDatasets WHERE FLsetupId = :FLsetupId AND FLpipeId = :FLpipeId"
+
+
+UPDATE_FLDATASET_QUERY = (
+    "UPDATE FedDatasets SET FLpipeId = :FLpipeId WHERE FedId = :FedId"
+)
+SELECT_FLDATASET_BY_NAME_QUERY = "SELECT * FROM FedDatasets WHERE name = :name"
+
+CREATE_FLPIPELINE_QUERY = """
+INSERT INTO FLpipeline (name, description, creation_date, results)
+VALUES ('{name}', '{description}', '{creation_date}', '{result}')
+"""
+DELETE_FLPIPELINE_QUERY = "DELETE FROM FLpipeline WHERE name = '{name}'"
+
+SELECT_FLPIPELINE_QUERY = "SELECT FROM FLpipeline WHERE name = '{name}'"
+
+CREATE_TEST_RESULTS_QUERY = """
+INSERT INTO testResults (pipelineid, nodename, confusionmatrix, accuracy , sensivity, ppv , npv , f1score , fpr , tpr )
+VALUES ('{pipelineId}', '{nodeName}', '{confusion_matrix}', '{accuracy}' , '{sensivity}' , '{ppv}' , '{npv}' , '{f1score}' , '{fpr}' , '{tpr}')
+"""

Medfl/NetManager/network.py

@@ -0,0 +1,160 @@
+# src/Medfl/NetManager/network.py
+
+from Medfl.LearningManager.utils import *
+
+from .net_helper import *
+from .net_manager_queries import (CREATE_MASTER_DATASET_TABLE_QUERY,
+                                  CREATE_DATASETS_TABLE_QUERY,
+                                  DELETE_NETWORK_QUERY,
+                                  INSERT_NETWORK_QUERY, LIST_ALL_NODES_QUERY,
+                                  UPDATE_NETWORK_QUERY, GET_NETWORK_QUERY)
+from .node import Node
+import pandas as pd
+from Medfl.LearningManager.utils import params
+
+
+class Network:
+    """
+    A class representing a network.
+
+    Attributes:
+        name (str): The name of the network.
+        mtable_exists (int): An integer flag indicating whether the MasterDataset table exists (1) or not (0).
+    """
+
+    def __init__(self, name: str = ""):
+        """
+        Initialize a Network instance.
+
+        Parameters:
+            name (str): The name of the network.
+        """
+        self.name = name
+        self.mtable_exists = int(master_table_exists())
+        self.validate()
+
+    def validate(self):
+        """Validate name"""
+
+        if not isinstance(self.name, str):
+            raise TypeError("name argument must be a string")
+
+    def create_network(self):
+        """Create a new network in the database."""
+        my_eng.execute(text(INSERT_NETWORK_QUERY.format(name=self.name)))
+        self.id = get_netid_from_name(self.name)
+
+    def use_network(self, network_name: str):
+        """Use a network in the database.
+
+        Parameters:
+            network_name (str): The name of the network to use.
+
+        Returns:
+            Network or None: An instance of the Network class if the network exists, else None.
+        
+        """
+        network = pd.read_sql(
+            text(GET_NETWORK_QUERY.format(name=network_name)),
+            my_eng,
+        )
+
+        if (network.NetId[0]):
+            self.name = network.NetName[0]
+            self.id = network.NetId[0]
+            self.mtable_exists = int(master_table_exists())
+            self.validate()
+            return self
+        else:
+            return None
+
+    def delete_network(self):
+        """Delete the network from the database."""
+        my_eng.execute(text(DELETE_NETWORK_QUERY.format(name=self.name)))
+
+    def update_network(self, FLsetupId: int):
+        """Update the network's FLsetupId in the database.
+        
+        Parameters:
+            FLsetupId (int): The FLsetupId to update.
+        """
+        my_eng.execute(
+            text(UPDATE_NETWORK_QUERY.format(FLsetupId=FLsetupId, id=self.id))
+        )
+
+    def add_node(self, node: Node):
+        """Add a node to the network.
+
+        Parameters:
+            node (Node): The node to add.
+        """
+        node.create_node(self.id)
+
+    def list_allnodes(self):
+        """List all nodes in the network.
+
+        Parameters:
+            None
+
+        Returns:
+            DataFrame: A DataFrame containing information about all nodes in the network.
+        
+        """
+        return pd.read_sql(
+            text(LIST_ALL_NODES_QUERY.format(name=self.name)), my_eng
+        )
+
+    def create_master_dataset(self, path_to_csv: str = params['path_to_master_csv']):
+        """
+        Create the MasterDataset table and insert dataset values.
+
+        :param path_to_csv: Path to the CSV file containing the dataset.
+        """
+        print(path_to_csv)
+        # Read the CSV file into a Pandas DataFrame
+        data_df = pd.read_csv(path_to_csv)
+
+        # Process the data if needed (e.g., handle missing values, encode categorical variables)
+        # ...
+
+        # Check if the MasterDataset table exists
+
+        if self.mtable_exists != 1:
+            columns = data_df.columns.tolist()
+            columns_str = ",\n".join(
+                [
+                    f"{col} {column_map[str(data_df[col].dtype)]}"
+                    for col in columns
+                ]
+            )
+            my_eng.execute(
+                text(CREATE_MASTER_DATASET_TABLE_QUERY.format(columns_str))
+            )
+            my_eng.execute(text(CREATE_DATASETS_TABLE_QUERY.format(columns_str)))
+
+            # Get the list of columns in the DataFrame
+
+            data_df = process_eicu(data_df)
+            # Insert the dataset values into the MasterDataset table
+
+            for index, row in data_df.iterrows():
+                query_1 = "INSERT INTO MasterDataset(" + "".join(
+                    f"{x}," for x in columns
+                )
+                query_2 = f"VALUES (" + "".join(
+                    f"{is_str(data_df, row, x)}," for x in columns
+                )
+                query = query_1[:-1] + ")" + query_2[:-1] + ")"
+                my_eng.execute(text(query))
+
+        # Set mtable_exists flag to True
+        self.mtable_exists = 1
+
+    @staticmethod
+    def list_allnetworks():
+        """List all networks in the database.
+        Returns:
+            DataFrame: A DataFrame containing information about all networks in the database.
+        
+        """
+        return pd.read_sql(text("SELECT * FROM Networks"), my_eng)