MEDfl 0.1.31__py3-none-any.whl → 0.1.33__py3-none-any.whl

Medfl/NetManager/net_manager_queries.py

@@ -1,137 +1,142 @@
-from .net_helper import is_str
-
-INSERT_DATASET = """
-    INSERT INTO DataSets(DataSetName, NodeId, {columns})
-    VALUES (:name, :NodeId, {values})
-"""
-DELETE_DATASET = """
-    DELETE FROM DataSets WHERE DataSetName = :name
-"""
-
-SELECT_ALL_DATASET_NAMES = """
-    SELECT DISTINCT DataSetName,NodeId FROM DataSets
-"""
-
-SELECT_DATASET_BY_NAME = """
-    SELECT * FROM DataSets WHERE DataSetName = :name
-"""
-
-# node queries
-# sql_queries.py
-
-INSERT_NODE_QUERY = (
-    "INSERT INTO Nodes(NodeName,NetId,train) VALUES ('{}',{}, {})"
-)
-DELETE_NODE_QUERY = "DELETE FROM Nodes WHERE NodeName = '{}'"
-SELECT_MASTER_COLUMNS_QUERY = "SELECT * FROM MasterDataset LIMIT 1"
-SELECT_DATASET_BY_COLUMN_QUERY = "SELECT * FROM MasterDataset WHERE {} = '{}'"
-SELECT_DATASET_BY_NODE_ID_QUERY = "SELECT * FROM DataSets WHERE NodeId = {}"
-
-SELECT_ALL_DATASETS_QUERY = "SELECT DISTINCT DataSetName,NodeName FROM DataSets,Nodes WHERE Nodes.NodeName = '{}' and Nodes.NodeId = DataSets.NodeId"
-SELECT_ALL_NODES_QUERY = "SELECT * FROM Nodes"
-
-
-# SQL query to insert a new network
-INSERT_NETWORK_QUERY = "INSERT INTO Networks(NetName) VALUES ('{name}')"
-
-# SQL query to delete a network
-DELETE_NETWORK_QUERY = "DELETE FROM Networks WHERE NetName = '{name}'"
-
-# SQL query to delete a network
-GET_NETWORK_QUERY = "SELECT * FROM Networks WHERE NetName = '{name}'"
-
-
-# SQL query to update a network
-UPDATE_NETWORK_QUERY = (
-    "UPDATE Networks SET FLsetupId = {FLsetupId} WHERE NetId = {id}"
-)
-
-# SQL query to retrieve all nodes for a network
-LIST_ALL_NODES_QUERY = "SELECT Nodes.NodeName, Networks.NetName FROM Nodes, Networks WHERE Networks.NetName = '{name}' AND Networks.NetId = Nodes.NetId"
-
-# SQL query to create the MasterDataset table
-CREATE_MASTER_DATASET_TABLE_QUERY = """
-CREATE TABLE IF NOT EXISTS MasterDataset (
-    PatientId INT NOT NULL AUTO_INCREMENT,
-    {},
-    PRIMARY KEY (PatientId)
-);
-"""
-
-# SQL query to create the datasets table
-CREATE_DATASETS_TABLE_QUERY = """
-CREATE TABLE IF NOT EXISTS Datasets (
-                     DataSetId INT NOT NULL AUTO_INCREMENT, 
-                     DataSetName VARCHAR(255), 
-                     NodeId INT,
-                     {},
-                     PRIMARY KEY (DataSetId)
-                     
-);
-"""
-
-# SQL query to insert dataset values
-INSERT_DATASET_VALUES_QUERY = "INSERT INTO MasterDataset({columns}, NodeId) VALUES ('{name}', {nodeId}, {values})"
-
-
-# FL setup_queries
-# sql_queries.py
-
-CREATE_FLSETUP_QUERY = """
-    INSERT INTO FLsetup (name, description, creation_date, NetId, column_name)
-    VALUES (:name, :description, :creation_date, :net_id, :column_name)
-"""
-
-DELETE_FLSETUP_QUERY = """
-    DELETE FROM FLsetup
-    WHERE name = :name
-"""
-
-UPDATE_FLSETUP_QUERY = UPDATE_NETWORK_QUERY = (
-    "UPDATE FLsetup SET column_name ='{column_name}' WHERE name ='{FLsetupName}'"
-)
-
-
-READ_SETUP_QUERY = """
-    SELECT * FROM FLsetup
-    WHERE FLsetupId = :flsetup_id
-"""
-
-READ_ALL_SETUPS_QUERY = """
-    SELECT * FROM FLsetup
-"""
-
-READ_NETWORK_BY_ID_QUERY = """
-    SELECT * FROM Networks
-    WHERE NetId = :net_id
-"""
-
-READ_DISTINCT_NODES_QUERY = """
-SELECT DISTINCT {} FROM MasterDataset 
-"""
-
-
-# FederatedDataset Queries
-INSERT_FLDATASET_QUERY = (
-    "INSERT INTO FedDatasets(name, FLsetupId) VALUES (:name, :FLsetupId)"
-)
-DELETE_FLDATASET_BY_SETUP_AND_PIPELINE_QUERY = "DELETE FROM FedDatasets WHERE FLsetupId = :FLsetupId AND FLpipeId = :FLpipeId"
-
-
-UPDATE_FLDATASET_QUERY = (
-    "UPDATE FedDatasets SET FLpipeId = :FLpipeId WHERE FedId = :FedId"
-)
-SELECT_FLDATASET_BY_NAME_QUERY = "SELECT * FROM FedDatasets WHERE name = :name"
-
-CREATE_FLPIPELINE_QUERY = """
-INSERT INTO FLpipeline (name, description, creation_date, results)
-VALUES ('{name}', '{description}', '{creation_date}', '{result}')
-"""
-DELETE_FLPIPELINE_QUERY = "DELETE FROM FLpipeline WHERE name = '{name}'"
-
-SELECT_FLPIPELINE_QUERY = "SELECT FROM FLpipeline WHERE name = '{name}'"
-
-CREATE_TEST_RESULTS_QUERY = """
-INSERT INTO testResults (pipelineid, nodename, confusionmatrix, accuracy , sensivity, ppv , npv , f1score , fpr , tpr )
-VALUES ('{pipelineId}', '{nodeName}', '{confusion_matrix}', '{accuracy}' , '{sensivity}' , '{ppv}' , '{npv}' , '{f1score}' , '{fpr}' , '{tpr}')
-"""
+from .net_helper import is_str
+
+INSERT_DATASET = """
+    INSERT INTO DataSets(DataSetName, NodeId, {columns})
+    VALUES (:name, :NodeId, {values})
+"""
+DELETE_DATASET = """
+    DELETE FROM DataSets WHERE DataSetName = :name
+"""
+
+SELECT_ALL_DATASET_NAMES = """
+    SELECT DISTINCT DataSetName,NodeId FROM DataSets
+"""
+
+SELECT_DATASET_BY_NAME = """
+    SELECT * FROM DataSets WHERE DataSetName = :name
+"""
+
+# node queries
+# sql_queries.py
+
+INSERT_NODE_QUERY = (
+    "INSERT INTO Nodes(NodeName,NetId,train) VALUES ('{}',{}, {})"
+)
+DELETE_NODE_QUERY = "DELETE FROM Nodes WHERE NodeName = '{}'"
+SELECT_MASTER_COLUMNS_QUERY = "SELECT * FROM MasterDataset LIMIT 1"
+SELECT_DATASET_BY_COLUMN_QUERY = "SELECT * FROM MasterDataset WHERE {} = '{}'"
+SELECT_DATASET_BY_NODE_ID_QUERY = "SELECT * FROM DataSets WHERE NodeId = {}"
+
+SELECT_ALL_DATASETS_QUERY = "SELECT DISTINCT DataSetName,NodeName FROM DataSets,Nodes WHERE Nodes.NodeName = '{}' and Nodes.NodeId = DataSets.NodeId"
+SELECT_ALL_NODES_QUERY = "SELECT * FROM Nodes"
+
+
+# SQL query to insert a new network
+INSERT_NETWORK_QUERY = "INSERT INTO Networks(NetName) VALUES (:name)"
+
+# SQL query to delete a network
+DELETE_NETWORK_QUERY = "DELETE FROM Networks WHERE NetName = '{name}'"
+
+# SQL query to delete a network
+GET_NETWORK_QUERY = "SELECT * FROM Networks WHERE NetName = '{name}'"
+
+
+# SQL query to update a network
+UPDATE_NETWORK_QUERY = (
+    "UPDATE Networks SET FLsetupId = {FLsetupId} WHERE NetId = {id}"
+)
+
+# SQL query to retrieve all nodes for a network
+LIST_ALL_NODES_QUERY = """
+SELECT Nodes.NodeName, Networks.NetName 
+FROM Nodes 
+JOIN Networks ON Networks.NetId = Nodes.NetId 
+WHERE Networks.NetName = :name
+"""
+
+# SQL query to create the MasterDataset table (SQLite-compatible)
+CREATE_MASTER_DATASET_TABLE_QUERY = """
+CREATE TABLE IF NOT EXISTS MasterDataset (
+    PatientId INTEGER PRIMARY KEY AUTOINCREMENT,
+    {}
+);
+"""
+
+
+# SQL query to create the datasets table (SQLite-compatible)
+CREATE_DATASETS_TABLE_QUERY = """
+CREATE TABLE IF NOT EXISTS Datasets (
+    DataSetId INTEGER PRIMARY KEY AUTOINCREMENT, 
+    DataSetName VARCHAR(255), 
+    NodeId INT,
+    {}
+);
+"""
+
+
+
+# SQL query to insert dataset values
+INSERT_DATASET_VALUES_QUERY = "INSERT INTO MasterDataset({columns}, NodeId) VALUES ('{name}', {nodeId}, {values})"
+
+
+# FL setup_queries
+# sql_queries.py
+
+CREATE_FLSETUP_QUERY = """
+    INSERT INTO FLsetup (name, description, creation_date, NetId, column_name)
+    VALUES (:name, :description, :creation_date, :net_id, :column_name)
+"""
+
+DELETE_FLSETUP_QUERY = """
+    DELETE FROM FLsetup
+    WHERE name = :name
+"""
+
+UPDATE_FLSETUP_QUERY = UPDATE_NETWORK_QUERY = (
+    "UPDATE FLsetup SET column_name ='{column_name}' WHERE name ='{FLsetupName}'"
+)
+
+
+READ_SETUP_QUERY = """
+    SELECT * FROM FLsetup
+    WHERE FLsetupId = :flsetup_id
+"""
+
+READ_ALL_SETUPS_QUERY = """
+    SELECT * FROM FLsetup
+"""
+
+READ_NETWORK_BY_ID_QUERY = """
+    SELECT * FROM Networks
+    WHERE NetId = :net_id
+"""
+
+READ_DISTINCT_NODES_QUERY = """
+SELECT DISTINCT {} FROM MasterDataset 
+"""
+
+
+# FederatedDataset Queries
+INSERT_FLDATASET_QUERY = (
+    "INSERT INTO FedDatasets(name, FLsetupId) VALUES (:name, :FLsetupId)"
+)
+DELETE_FLDATASET_BY_SETUP_AND_PIPELINE_QUERY = "DELETE FROM FedDatasets WHERE FLsetupId = :FLsetupId AND FLpipeId = :FLpipeId"
+
+
+UPDATE_FLDATASET_QUERY = (
+    "UPDATE FedDatasets SET FLpipeId = :FLpipeId WHERE FedId = :FedId"
+)
+SELECT_FLDATASET_BY_NAME_QUERY = "SELECT * FROM FedDatasets WHERE name = :name"
+
+CREATE_FLPIPELINE_QUERY = """
+INSERT INTO FLpipeline (name, description, creation_date, results)
+VALUES ('{name}', '{description}', '{creation_date}', '{result}')
+"""
+DELETE_FLPIPELINE_QUERY = "DELETE FROM FLpipeline WHERE name = '{name}'"
+
+SELECT_FLPIPELINE_QUERY = "SELECT FROM FLpipeline WHERE name = '{name}'"
+
+CREATE_TEST_RESULTS_QUERY = """
+INSERT INTO testResults (pipelineid, nodename, confusionmatrix, accuracy , sensivity, ppv , npv , f1score , fpr , tpr )
+VALUES ('{pipelineId}', '{nodeName}', '{confusion_matrix}', '{accuracy}' , '{sensivity}' , '{ppv}' , '{npv}' , '{f1score}' , '{fpr}' , '{tpr}')
+"""

Medfl/NetManager/network.py

@@ -1,174 +1,194 @@
-# src/MEDfl/NetManager/network.py
-
-from MEDfl.LearningManager.utils import *
-
-from .net_helper import *
-from .net_manager_queries import (CREATE_MASTER_DATASET_TABLE_QUERY,
-                                  CREATE_DATASETS_TABLE_QUERY,
-                                  DELETE_NETWORK_QUERY,
-                                  INSERT_NETWORK_QUERY, LIST_ALL_NODES_QUERY,
-                                  UPDATE_NETWORK_QUERY, GET_NETWORK_QUERY)
-from .node import Node
-import pandas as pd
-from MEDfl.LearningManager.utils import params
-
-from sqlalchemy import text
-
-
-class Network:
-    """
-    A class representing a network.
-
-    Attributes:
-        name (str): The name of the network.
-        mtable_exists (int): An integer flag indicating whether the MasterDataset table exists (1) or not (0).
-    """
-
-    def __init__(self, name: str = ""):
-        """
-        Initialize a Network instance.
-
-        Parameters:
-            name (str): The name of the network.
-        """
-        self.name = name
-        self.mtable_exists = int(master_table_exists())
-        self.validate()
-
-        db_manager = DatabaseManager() ; 
-        db_manager.connect() ; 
-        self.eng = db_manager.get_connection()
-
-    def validate(self):
-        """Validate name"""
-
-        if not isinstance(self.name, str):
-            raise TypeError("name argument must be a string")
-
-    def create_network(self):
-        """Create a new network in the database."""
-        self.eng.execute(text(INSERT_NETWORK_QUERY.format(name=self.name)))
-        self.id = get_netid_from_name(self.name)
-
-    def use_network(self, network_name: str):
-        """Use a network in the database.
-
-        Parameters:
-            network_name (str): The name of the network to use.
-
-        Returns:
-            Network or None: An instance of the Network class if the network exists, else None.
-        
-        """
-        network = pd.read_sql(
-            text(GET_NETWORK_QUERY.format(name=network_name)),
-            self.eng,
-        )
-
-        if (network.NetId[0]):
-            self.name = network.NetName[0]
-            self.id = network.NetId[0]
-            self.mtable_exists = int(master_table_exists())
-            self.validate()
-            return self
-        else:
-            return None
-
-    def delete_network(self):
-        """Delete the network from the database."""
-        self.eng.execute(text(DELETE_NETWORK_QUERY.format(name=self.name)))
-
-    def update_network(self, FLsetupId: int):
-        """Update the network's FLsetupId in the database.
-        
-        Parameters:
-            FLsetupId (int): The FLsetupId to update.
-        """
-        self.eng.execute(
-            text(UPDATE_NETWORK_QUERY.format(FLsetupId=FLsetupId, id=self.id))
-        )
-
-    def add_node(self, node: Node):
-        """Add a node to the network.
-
-        Parameters:
-            node (Node): The node to add.
-        """
-        node.create_node(self.id)
-
-    def list_allnodes(self):
-        """List all nodes in the network.
-
-        Parameters:
-            None
-
-        Returns:
-            DataFrame: A DataFrame containing information about all nodes in the network.
-        
-        """
-        query = text(LIST_ALL_NODES_QUERY.format(name=self.name))
-        result_proxy = self.eng.execute(query)
-        result_df = pd.DataFrame(result_proxy.fetchall(), columns=result_proxy.keys())
-        return result_df
-
-    def create_master_dataset(self, path_to_csv: str = params['path_to_master_csv']):
-        """
-        Create the MasterDataset table and insert dataset values.
-
-        :param path_to_csv: Path to the CSV file containing the dataset.
-        """
-        print(path_to_csv)
-        # Read the CSV file into a Pandas DataFrame
-        data_df = pd.read_csv(path_to_csv)
-
-        # Process the data if needed (e.g., handle missing values, encode categorical variables)
-        # ...
-
-        # Check if the MasterDataset table exists
-
-        if self.mtable_exists != 1:
-            columns = data_df.columns.tolist()
-            columns_str = ",\n".join(
-                [
-                    f"{col} {column_map[str(data_df[col].dtype)]}"
-                    for col in columns
-                ]
-            )
-            self.eng.execute(
-                text(CREATE_MASTER_DATASET_TABLE_QUERY.format(columns_str))
-            )
-            self.eng.execute(text(CREATE_DATASETS_TABLE_QUERY.format(columns_str)))
-
-            # Get the list of columns in the DataFrame
-
-            data_df = process_eicu(data_df)
-            # Insert the dataset values into the MasterDataset table
-
-            for index, row in data_df.iterrows():
-                query_1 = "INSERT INTO MasterDataset(" + "".join(
-                    f"{x}," for x in columns
-                )
-                query_2 = f"VALUES (" + "".join(
-                    f"{is_str(data_df, row, x)}," for x in columns
-                )
-                query = query_1[:-1] + ")" + query_2[:-1] + ")"
-                self.eng.execute(text(query))
-
-        # Set mtable_exists flag to True
-        self.mtable_exists = 1
-
-    @staticmethod
-    def list_allnetworks():
-        """List all networks in the database.
-        Returns:
-            DataFrame: A DataFrame containing information about all networks in the database.
-        
-        """
-        db_manager = DatabaseManager() ; 
-        db_manager.connect() ; 
-        my_eng = db_manager.get_connection() ;
-
-        result_proxy = my_eng.execute("SELECT * FROM Networks")
-        result = result_proxy.fetchall()
-        return result
-
+# src/MEDfl/NetManager/network.py
+
+from MEDfl.LearningManager.utils import *
+from .net_helper import *
+from .net_manager_queries import (CREATE_MASTER_DATASET_TABLE_QUERY,
+                                  CREATE_DATASETS_TABLE_QUERY,
+                                  DELETE_NETWORK_QUERY,
+                                  INSERT_NETWORK_QUERY, LIST_ALL_NODES_QUERY,
+                                  UPDATE_NETWORK_QUERY, GET_NETWORK_QUERY)
+from .node import Node
+import pandas as pd
+from MEDfl.LearningManager.utils import params
+
+from sqlalchemy import text
+from sqlalchemy.exc import SQLAlchemyError
+
+class Network:
+    """
+    A class representing a network.
+
+    Attributes:
+        name (str): The name of the network.
+        mtable_exists (int): An integer flag indicating whether the MasterDataset table exists (1) or not (0).
+    """
+
+    def __init__(self, name: str = ""):
+        """
+        Initialize a Network instance.
+
+        Parameters:
+            name (str): The name of the network.
+        """
+        self.name = name
+        self.mtable_exists = int(master_table_exists())
+        self.validate()
+
+        db_manager = DatabaseManager() 
+        db_manager.connect() 
+        self.eng = db_manager.get_connection()
+
+    def validate(self):
+        """Validate name"""
+
+        if not isinstance(self.name, str):
+            raise TypeError("name argument must be a string")
+
+    def create_network(self):
+        """Create a new network in the database."""
+        try:
+            print(self.name)
+            self.eng.execute(text(INSERT_NETWORK_QUERY), {"name": self.name})
+            self.id = self.get_netid_from_name(self.name)
+        except SQLAlchemyError as e:
+            print(f"Error creating network: {e}")
+
+    def use_network(self, network_name: str):
+        """Use a network in the database.
+
+        Parameters:
+            network_name (str): The name of the network to use.
+
+        Returns:
+            Network or None: An instance of the Network class if the network exists, else None.
+        """
+        try:
+            network = pd.read_sql(
+                text(GET_NETWORK_QUERY),
+                self.eng,
+                params={"name": network_name}
+            )
+            if not network.empty:
+                self.name = network.iloc[0]['NetName']
+                self.id = network.iloc[0]['NetId']
+                self.mtable_exists = int(master_table_exists())
+                self.validate()
+                return self
+            else:
+                return None
+        except SQLAlchemyError as e:
+            print(f"Error using network: {e}")
+            return None
+
+    def delete_network(self):
+        """Delete the network from the database."""
+        try:
+            self.eng.execute(text(DELETE_NETWORK_QUERY), {"name": self.name})
+        except SQLAlchemyError as e:
+            print(f"Error deleting network: {e}")
+
+    def update_network(self, FLsetupId: int):
+        """Update the network's FLsetupId in the database.
+
+        Parameters:
+            FLsetupId (int): The FLsetupId to update.
+        """
+        try:
+            self.eng.execute(
+                text(UPDATE_NETWORK_QUERY),
+                {"FLsetupId": FLsetupId, "id": self.id}
+            )
+        except SQLAlchemyError as e:
+            print(f"Error updating network: {e}")
+
+    def add_node(self, node: Node):
+        """Add a node to the network.
+
+        Parameters:
+            node (Node): The node to add.
+        """
+        node.create_node(self.id)
+
+    def list_allnodes(self):
+        """List all nodes in the network.
+
+        Returns:
+            DataFrame: A DataFrame containing information about all nodes in the network.
+        """
+        try:
+            query = text(LIST_ALL_NODES_QUERY)
+            result_proxy = self.eng.execute(query, name=self.name)
+            result_df = pd.DataFrame(result_proxy.fetchall(), columns=result_proxy.keys())
+            return result_df
+        except SQLAlchemyError as e:
+            print(f"Error listing all nodes: {e}")
+            return pd.DataFrame()
+
+    def create_master_dataset(self, path_to_csv: str = params['path_to_master_csv']):
+        """
+        Create the MasterDataset table and insert dataset values.
+
+        :param path_to_csv: Path to the CSV file containing the dataset.
+        """
+        try:
+            print(path_to_csv)
+            data_df = pd.read_csv(path_to_csv)
+
+            if self.mtable_exists != 1:
+                columns = data_df.columns.tolist()
+                columns_str = ",\n".join(
+                    [
+                        f"{col} {column_map[str(data_df[col].dtype)]}"
+                        for col in columns
+                    ]
+                )
+                self.eng.execute(
+                    text(CREATE_MASTER_DATASET_TABLE_QUERY.format(columns_str))
+                )
+                self.eng.execute(text(CREATE_DATASETS_TABLE_QUERY.format(columns_str)))
+
+                # Process data
+                data_df = process_eicu(data_df)
+
+                # Insert data in batches
+                batch_size = 1000  # Adjust as needed
+                for start_idx in range(0, len(data_df), batch_size):
+                    batch_data = data_df.iloc[start_idx:start_idx + batch_size]
+                    insert_query = f"INSERT INTO MasterDataset ({', '.join(columns)}) VALUES ({', '.join([':' + col for col in columns])})"
+                    data_to_insert = batch_data.to_dict(orient='records')
+                    self.eng.execute(text(insert_query), data_to_insert)
+
+                self.mtable_exists = 1
+        except SQLAlchemyError as e:
+            print(f"Error creating master dataset: {e}")
+            
+    @staticmethod
+    def list_allnetworks():
+        """List all networks in the database.
+
+        Returns:
+            DataFrame: A DataFrame containing information about all networks in the database.
+        """
+        try:
+            db_manager = DatabaseManager() 
+            db_manager.connect() 
+            my_eng = db_manager.get_connection() 
+
+            result_proxy = my_eng.execute("SELECT * FROM Networks")
+            result = result_proxy.fetchall()
+            return pd.DataFrame(result, columns=result_proxy.keys())
+        except SQLAlchemyError as e:
+            print(f"Error listing all networks: {e}")
+            return pd.DataFrame()
+    
+    def get_netid_from_name(self, name):
+        """Get network ID from network name."""
+        try:
+            result = self.eng.execute(text("SELECT NetId FROM Networks WHERE NetName = :name"), {"name": name}).fetchone()
+            if result:
+                return result[0]
+            else:
+                return None
+        except SQLAlchemyError as e:
+            print(f"Error fetching network ID: {e}")
+            return None