MEDfl 2.0.4.dev0__py3-none-any.whl → 2.0.4.dev2__py3-none-any.whl

Medfl/NetManager/net_manager_queries.py

@@ -1,142 +0,0 @@
-from .net_helper import is_str
-
-INSERT_DATASET = """
-    INSERT INTO DataSets(DataSetName, NodeId, {columns})
-    VALUES (:name, :NodeId, {values})
-"""
-DELETE_DATASET = """
-    DELETE FROM DataSets WHERE DataSetName = :name
-"""
-
-SELECT_ALL_DATASET_NAMES = """
-    SELECT DISTINCT DataSetName,NodeId FROM DataSets
-"""
-
-SELECT_DATASET_BY_NAME = """
-    SELECT * FROM DataSets WHERE DataSetName = :name
-"""
-
-# node queries
-# sql_queries.py
-
-INSERT_NODE_QUERY = (
-    "INSERT INTO Nodes(NodeName,NetId,train) VALUES ('{}',{}, {})"
-)
-DELETE_NODE_QUERY = "DELETE FROM Nodes WHERE NodeName = '{}'"
-SELECT_MASTER_COLUMNS_QUERY = "SELECT * FROM MasterDataset LIMIT 1"
-SELECT_DATASET_BY_COLUMN_QUERY = "SELECT * FROM MasterDataset WHERE {} = '{}'"
-SELECT_DATASET_BY_NODE_ID_QUERY = "SELECT * FROM DataSets WHERE NodeId = {}"
-
-SELECT_ALL_DATASETS_QUERY = "SELECT DISTINCT DataSetName,NodeName FROM DataSets,Nodes WHERE Nodes.NodeName = '{}' and Nodes.NodeId = DataSets.NodeId"
-SELECT_ALL_NODES_QUERY = "SELECT * FROM Nodes"
-
-
-# SQL query to insert a new network
-INSERT_NETWORK_QUERY = "INSERT INTO Networks(NetName) VALUES (:name)"
-
-# SQL query to delete a network
-DELETE_NETWORK_QUERY = "DELETE FROM Networks WHERE NetName = '{name}'"
-
-# SQL query to delete a network
-GET_NETWORK_QUERY = "SELECT * FROM Networks WHERE NetName = '{name}'"
-
-
-# SQL query to update a network
-UPDATE_NETWORK_QUERY = (
-    "UPDATE Networks SET FLsetupId = {FLsetupId} WHERE NetId = {id}"
-)
-
-# SQL query to retrieve all nodes for a network
-LIST_ALL_NODES_QUERY = """
-SELECT Nodes.NodeName, Networks.NetName 
-FROM Nodes 
-JOIN Networks ON Networks.NetId = Nodes.NetId 
-WHERE Networks.NetName = :name
-"""
-
-# SQL query to create the MasterDataset table (SQLite-compatible)
-CREATE_MASTER_DATASET_TABLE_QUERY = """
-CREATE TABLE IF NOT EXISTS MasterDataset (
-    PatientId INTEGER PRIMARY KEY AUTOINCREMENT,
-    {}
-);
-"""
-
-
-# SQL query to create the datasets table (SQLite-compatible)
-CREATE_DATASETS_TABLE_QUERY = """
-CREATE TABLE IF NOT EXISTS Datasets (
-    DataSetId INTEGER PRIMARY KEY AUTOINCREMENT, 
-    DataSetName VARCHAR(255), 
-    NodeId INT,
-    {}
-);
-"""
-
-
-
-# SQL query to insert dataset values
-INSERT_DATASET_VALUES_QUERY = "INSERT INTO MasterDataset({columns}, NodeId) VALUES ('{name}', {nodeId}, {values})"
-
-
-# FL setup_queries
-# sql_queries.py
-
-CREATE_FLSETUP_QUERY = """
-    INSERT INTO FLsetup (name, description, creation_date, NetId, column_name)
-    VALUES (:name, :description, :creation_date, :net_id, :column_name)
-"""
-
-DELETE_FLSETUP_QUERY = """
-    DELETE FROM FLsetup
-    WHERE name = :name
-"""
-
-UPDATE_FLSETUP_QUERY = UPDATE_NETWORK_QUERY = (
-    "UPDATE FLsetup SET column_name ='{column_name}' WHERE name ='{FLsetupName}'"
-)
-
-
-READ_SETUP_QUERY = """
-    SELECT * FROM FLsetup
-    WHERE FLsetupId = :flsetup_id
-"""
-
-READ_ALL_SETUPS_QUERY = """
-    SELECT * FROM FLsetup
-"""
-
-READ_NETWORK_BY_ID_QUERY = """
-    SELECT * FROM Networks
-    WHERE NetId = :net_id
-"""
-
-READ_DISTINCT_NODES_QUERY = """
-SELECT DISTINCT {} FROM MasterDataset 
-"""
-
-
-# FederatedDataset Queries
-INSERT_FLDATASET_QUERY = (
-    "INSERT INTO FedDatasets(name, FLsetupId) VALUES (:name, :FLsetupId)"
-)
-DELETE_FLDATASET_BY_SETUP_AND_PIPELINE_QUERY = "DELETE FROM FedDatasets WHERE FLsetupId = :FLsetupId AND FLpipeId = :FLpipeId"
-
-
-UPDATE_FLDATASET_QUERY = (
-    "UPDATE FedDatasets SET FLpipeId = :FLpipeId WHERE FedId = :FedId"
-)
-SELECT_FLDATASET_BY_NAME_QUERY = "SELECT * FROM FedDatasets WHERE name = :name"
-
-CREATE_FLPIPELINE_QUERY = """
-INSERT INTO FLpipeline (name, description, creation_date, results)
-VALUES ('{name}', '{description}', '{creation_date}', '{result}')
-"""
-DELETE_FLPIPELINE_QUERY = "DELETE FROM FLpipeline WHERE name = '{name}'"
-
-SELECT_FLPIPELINE_QUERY = "SELECT FROM FLpipeline WHERE name = '{name}'"
-
-CREATE_TEST_RESULTS_QUERY = """
-INSERT INTO testResults (pipelineid, nodename, confusionmatrix, accuracy , sensivity, ppv , npv , f1score , fpr , tpr )
-VALUES ('{pipelineId}', '{nodeName}', '{confusion_matrix}', '{accuracy}' , '{sensivity}' , '{ppv}' , '{npv}' , '{f1score}' , '{fpr}' , '{tpr}')
-"""

Medfl/NetManager/network.py

@@ -1,194 +0,0 @@
-# src/MEDfl/NetManager/network.py
-
-from MEDfl.LearningManager.utils import *
-from .net_helper import *
-from .net_manager_queries import (CREATE_MASTER_DATASET_TABLE_QUERY,
-                                  CREATE_DATASETS_TABLE_QUERY,
-                                  DELETE_NETWORK_QUERY,
-                                  INSERT_NETWORK_QUERY, LIST_ALL_NODES_QUERY,
-                                  UPDATE_NETWORK_QUERY, GET_NETWORK_QUERY)
-from .node import Node
-import pandas as pd
-from MEDfl.LearningManager.utils import params
-
-from sqlalchemy import text
-from sqlalchemy.exc import SQLAlchemyError
-
-class Network:
-    """
-    A class representing a network.
-
-    Attributes:
-        name (str): The name of the network.
-        mtable_exists (int): An integer flag indicating whether the MasterDataset table exists (1) or not (0).
-    """
-
-    def __init__(self, name: str = ""):
-        """
-        Initialize a Network instance.
-
-        Parameters:
-            name (str): The name of the network.
-        """
-        self.name = name
-        self.mtable_exists = int(master_table_exists())
-        self.validate()
-
-        db_manager = DatabaseManager() 
-        db_manager.connect() 
-        self.eng = db_manager.get_connection()
-
-    def validate(self):
-        """Validate name"""
-
-        if not isinstance(self.name, str):
-            raise TypeError("name argument must be a string")
-
-    def create_network(self):
-        """Create a new network in the database."""
-        try:
-            print(self.name)
-            self.eng.execute(text(INSERT_NETWORK_QUERY), {"name": self.name})
-            self.id = self.get_netid_from_name(self.name)
-        except SQLAlchemyError as e:
-            print(f"Error creating network: {e}")
-
-    def use_network(self, network_name: str):
-        """Use a network in the database.
-
-        Parameters:
-            network_name (str): The name of the network to use.
-
-        Returns:
-            Network or None: An instance of the Network class if the network exists, else None.
-        """
-        try:
-            network = pd.read_sql(
-                text(GET_NETWORK_QUERY),
-                self.eng,
-                params={"name": network_name}
-            )
-            if not network.empty:
-                self.name = network.iloc[0]['NetName']
-                self.id = network.iloc[0]['NetId']
-                self.mtable_exists = int(master_table_exists())
-                self.validate()
-                return self
-            else:
-                return None
-        except SQLAlchemyError as e:
-            print(f"Error using network: {e}")
-            return None
-
-    def delete_network(self):
-        """Delete the network from the database."""
-        try:
-            self.eng.execute(text(DELETE_NETWORK_QUERY), {"name": self.name})
-        except SQLAlchemyError as e:
-            print(f"Error deleting network: {e}")
-
-    def update_network(self, FLsetupId: int):
-        """Update the network's FLsetupId in the database.
-
-        Parameters:
-            FLsetupId (int): The FLsetupId to update.
-        """
-        try:
-            self.eng.execute(
-                text(UPDATE_NETWORK_QUERY),
-                {"FLsetupId": FLsetupId, "id": self.id}
-            )
-        except SQLAlchemyError as e:
-            print(f"Error updating network: {e}")
-
-    def add_node(self, node: Node):
-        """Add a node to the network.
-
-        Parameters:
-            node (Node): The node to add.
-        """
-        node.create_node(self.id)
-
-    def list_allnodes(self):
-        """List all nodes in the network.
-
-        Returns:
-            DataFrame: A DataFrame containing information about all nodes in the network.
-        """
-        try:
-            query = text(LIST_ALL_NODES_QUERY)
-            result_proxy = self.eng.execute(query, name=self.name)
-            result_df = pd.DataFrame(result_proxy.fetchall(), columns=result_proxy.keys())
-            return result_df
-        except SQLAlchemyError as e:
-            print(f"Error listing all nodes: {e}")
-            return pd.DataFrame()
-
-    def create_master_dataset(self, path_to_csv: str = params['path_to_master_csv']):
-        """
-        Create the MasterDataset table and insert dataset values.
-
-        :param path_to_csv: Path to the CSV file containing the dataset.
-        """
-        try:
-            print(path_to_csv)
-            data_df = pd.read_csv(path_to_csv)
-
-            if self.mtable_exists != 1:
-                columns = data_df.columns.tolist()
-                columns_str = ",\n".join(
-                    [
-                        f"{col} {column_map[str(data_df[col].dtype)]}"
-                        for col in columns
-                    ]
-                )
-                self.eng.execute(
-                    text(CREATE_MASTER_DATASET_TABLE_QUERY.format(columns_str))
-                )
-                self.eng.execute(text(CREATE_DATASETS_TABLE_QUERY.format(columns_str)))
-
-                # Process data
-                data_df = process_eicu(data_df)
-
-                # Insert data in batches
-                batch_size = 1000  # Adjust as needed
-                for start_idx in range(0, len(data_df), batch_size):
-                    batch_data = data_df.iloc[start_idx:start_idx + batch_size]
-                    insert_query = f"INSERT INTO MasterDataset ({', '.join(columns)}) VALUES ({', '.join([':' + col for col in columns])})"
-                    data_to_insert = batch_data.to_dict(orient='records')
-                    self.eng.execute(text(insert_query), data_to_insert)
-
-                self.mtable_exists = 1
-        except SQLAlchemyError as e:
-            print(f"Error creating master dataset: {e}")
-            
-    @staticmethod
-    def list_allnetworks():
-        """List all networks in the database.
-
-        Returns:
-            DataFrame: A DataFrame containing information about all networks in the database.
-        """
-        try:
-            db_manager = DatabaseManager() 
-            db_manager.connect() 
-            my_eng = db_manager.get_connection() 
-
-            result_proxy = my_eng.execute("SELECT * FROM Networks")
-            result = result_proxy.fetchall()
-            return pd.DataFrame(result, columns=result_proxy.keys())
-        except SQLAlchemyError as e:
-            print(f"Error listing all networks: {e}")
-            return pd.DataFrame()
-    
-    def get_netid_from_name(self, name):
-        """Get network ID from network name."""
-        try:
-            result = self.eng.execute(text("SELECT NetId FROM Networks WHERE NetName = :name"), {"name": name}).fetchone()
-            if result:
-                return result[0]
-            else:
-                return None
-        except SQLAlchemyError as e:
-            print(f"Error fetching network ID: {e}")
-            return None

Medfl/NetManager/node.py

@@ -1,184 +0,0 @@
-import pandas as pd
-
-from .net_helper import *
-from .net_manager_queries import *
-from MEDfl.LearningManager.utils import params
-from MEDfl.NetManager.database_connector import DatabaseManager
-
-from sqlalchemy import text, exc
-
-
-class Node:
-    """
-    A class representing a node in the network.
-
-    Attributes:
-        name (str): The name of the node.
-        train (int): An integer flag representing whether the node is used for training (1) or testing (0).
-        test_fraction (float, optional): The fraction of data used for testing when train=1. Default is 0.2.
-    """
-
-    def __init__(
-        self, name: str, train: int, test_fraction: float = 0.2, engine=None
-    ):
-        """
-        Initialize a Node instance.
-
-        Parameters:
-            name (str): The name of the node.
-            train (int): An integer flag representing whether the node is used for training (1) or testing (0).
-            test_fraction (float, optional): The fraction of data used for testing when train=1. Default is 0.2.
-        """
-        self.name = name
-        self.train = train
-        self.test_fraction = 1.0 if self.train == 0 else test_fraction
-       
-
-        db_manager = DatabaseManager() ; 
-        db_manager.connect() ; 
-        self.engine = db_manager.get_connection()
-
-    def validate(self):
-        """Validate name, train, test_fraction"""
-        if not isinstance(self.name, str):
-            raise TypeError("name argument must be a string")
-
-        if not isinstance(self.train, int):
-            raise TypeError("train argument must be an int")
-
-        if not isinstance(self.test_fraction, float):
-            raise TypeError("test_fraction argument must be a float")
-
-    def create_node(self, NetId: int):
-        """Create a node in the database.
-        Parameters:
-            NetId (int): The ID of the network to which the node belongs.
-
-        Returns:
-            None
-        """
-        self.engine.execute(
-            text(INSERT_NODE_QUERY.format(self.name, NetId, self.train))
-        )
-
-    def delete_node(self):
-        """Delete the node from the database."""
-        self.engine.execute(text(DELETE_NODE_QUERY.format(self.name)))
-
-    def check_dataset_compatibility(self, data_df):
-        """Check if the dataset is compatible with the master dataset.
-        Parameters:
-            data_df (DataFrame): The dataset to check.
-
-        Returns:
-            None
-        """
-        if master_table_exists() != 1:
-            print("MasterDataset doesn't exist")
-        else:
-            columns = data_df.columns.tolist()
-            
-            # get master_dataset columns
-            result_proxy = self.engine.execute(SELECT_MASTER_COLUMNS_QUERY)
-            master_table_columns = result_proxy.keys()
-
-
-            assert [x == y for x, y in zip(master_table_columns, columns)]
-
-    def update_node(self):
-        """Update the node information (not implemented)."""
-        pass
-
-    def get_dataset(self, column_name: str = None):
-        """Get the dataset for the node based on the given column name.
-        Parameters:
-            column_name (str, optional): The column name to filter the dataset. Default is None.
-
-        Returns:
-            DataFrame: The dataset associated with the node.
-        """
-        NodeId = get_nodeid_from_name(self.name)
-        if column_name is not None:
-            query = text(SELECT_DATASET_BY_COLUMN_QUERY.format(column_name, self.name))
-        else:
-            query = text(SELECT_DATASET_BY_NODE_ID_QUERY.format(NodeId))
-
-        result_proxy = self.engine.execute(query)
-        node_dataset = pd.DataFrame(result_proxy.fetchall(), columns=result_proxy.keys())
-
-        return node_dataset
-
-    def upload_dataset(self, dataset_name: str, path_to_csv: str = params['path_to_test_csv']):
-        """Upload the dataset to the database for the node.
-
-        Parameters:
-            dataset_name (str): The name of the dataset.
-            path_to_csv (str, optional): Path to the CSV file containing the dataset. Default is the path in params.
-
-        Returns:
-            None
-        """
-        try:
-            data_df = pd.read_csv(path_to_csv)
-            nodeId = get_nodeid_from_name(self.name)
-            columns = data_df.columns.tolist()
-            self.check_dataset_compatibility(data_df)
-
-            data_df = process_eicu(data_df)
-
-            # Insert data in batches
-            batch_size = 1000  # Adjust as needed
-            for start_idx in range(0, len(data_df), batch_size):
-                batch_data = data_df.iloc[start_idx:start_idx + batch_size]
-                insert_query = f"INSERT INTO Datasets (DataSetName, NodeId, {', '.join(columns)}) VALUES (:dataset_name, :nodeId, {', '.join([':' + col for col in columns])})"
-                data_to_insert = batch_data.to_dict(orient='records')
-                params = [{"dataset_name": dataset_name, "nodeId": nodeId, **row} for row in data_to_insert]
-                self.engine.execute(text(insert_query), params)
-        except exc.SQLAlchemyError as e:
-            print(f"Error uploading dataset: {e}")
-
-    def assign_dataset(self, dataset_name:str):
-        """Assigning existing dataSet to node
-        Parameters:
-            dataset_name (str): The name of the dataset to assign.
-
-        Returns:
-            None
-        """
-
-        nodeId = get_nodeid_from_name(self.name)
-        query = f"UPDATE DataSets SET nodeId = {nodeId} WHERE DataSetName = '{dataset_name}'"
-        self.engine.execute(text(query))
-
-    def unassign_dataset(self, dataset_name:str):
-        """unssigning existing dataSet to node
-        Parameters:
-            dataset_name (str): The name of the dataset to assign.
-
-        Returns:
-            None
-        """
-
-        query = f"UPDATE DataSets SET nodeId = {-1} WHERE DataSetName = '{dataset_name}'"
-        self.engine.execute(text(query))
-
-    def list_alldatasets(self):
-        """List all datasets associated with the node.
-        Returns:
-            DataFrame: A DataFrame containing information about all datasets associated with the node.
-        
-        """
-        return pd.read_sql(
-            text(SELECT_ALL_DATASETS_QUERY.format(self.name)), my_eng
-        )
-
-    @staticmethod
-    def list_allnodes():
-        """List all nodes in the database.
-        Returns:
-            DataFrame: A DataFrame containing information about all nodes in the database.
-        
-        """
-        query = text(SELECT_ALL_NODES_QUERY)
-        res = pd.read_sql(query, my_eng)
-        return res

Medfl/__init__.py

@@ -1,3 +0,0 @@
-from .LearningManager import *
-from .NetManager import *
-from .scripts import *

Medfl/scripts/__init__.py

@@ -1,2 +0,0 @@
-# from .base import *
-# from .create_db import *

Medfl/scripts/base.py

@@ -1,30 +0,0 @@
-import mysql.connector
-from sqlalchemy import create_engine, text
-from configparser import ConfigParser
-import yaml
-import pkg_resources
-import os
-
-# Get the directory of the current script
-current_directory = os.path.dirname(os.path.abspath(__file__))
-
-# Load configuration from the config file
-config_file_path = os.path.join(current_directory, 'db_config.ini')
-
-config = ConfigParser()
-config.read(config_file_path)
-mysql_config = config['mysql']
-
-
-
-connection_string = (
-    f"mysql+mysqlconnector://{mysql_config['user']}:{mysql_config['password']}@"
-    f"{mysql_config['host']}:{mysql_config['port']}/{mysql_config['database']}"
-)
-
-eng = create_engine(
-    connection_string,
-    execution_options={"autocommit": True},
-)
-
-my_eng = eng.connect()

Medfl/scripts/create_db.py

@@ -1,126 +0,0 @@
-import sys
-import sqlite3
-import pandas as pd
-from configparser import ConfigParser
-import os
-import ast 
-
-from MEDfl.LearningManager.utils import *
-
-
-def main(csv_file_path):
-    try:
-        # Get the directory of the current script
-        current_directory = os.path.dirname(os.path.abspath(__file__))
-
-        # Load configuration from the config file
-        # config_file_path = os.path.join(current_directory, 'sqllite_config.ini')*
-
-        config_file_path = load_db_config()
-
-        # config = ConfigParser()
-        # config.read(config_file_path)
-        # sqlite_config = config['sqllite']
-
-        sqlite_config = config_file_path 
-
-
-        print('Im here !')
-
-        # Connect to SQLite database (it will be created if it doesn't exist)
-        database_path = sqlite_config['database']
-        conn = sqlite3.connect(database_path)
-        cursor = conn.cursor()
-
-        # Drop each table if it exists
-        tables = ['Networks', 'FLsetup', 'Nodes', 'DataSets', 'FLpipeline', 'testResults', 'FedDatasets']
-        for table in tables:
-            cursor.execute(f"DROP TABLE IF EXISTS {table}")
-
-        # Create Networks table
-        cursor.execute(
-            "CREATE TABLE Networks( \
-                         NetId INTEGER PRIMARY KEY AUTOINCREMENT, \
-                         NetName TEXT \
-                         );"
-        )
-
-        # Create FLsetup table
-        cursor.execute("CREATE TABLE FLsetup (\
-        FLsetupId INTEGER PRIMARY KEY AUTOINCREMENT,\
-        name TEXT NOT NULL, \
-        description TEXT NOT NULL,\
-        creation_date TEXT NOT NULL,\
-        NetId INTEGER NOT NULL,\
-        column_name TEXT\
-        )")
-
-        # Create Nodes table
-        cursor.execute("CREATE TABLE Nodes ( \
-         NodeId INTEGER PRIMARY KEY AUTOINCREMENT,\
-         NodeName TEXT,\
-         train BOOLEAN DEFAULT 1,\
-         NetId INTEGER\
-        )")
-
-        data_df = pd.read_csv(csv_file_path)
-        columns = data_df.columns.tolist()
-        column_map = {"object": "TEXT", "int64": "INTEGER", "float64": "REAL"}
-        sub_query = ", ".join(f"{col} {column_map[str(data_df[col].dtype)]}" for col in columns)
-
-        # Create Datasets table by getting columns from the master csv file
-        cursor.execute(
-            f"CREATE TABLE DataSets( \
-                         DataSetId INTEGER PRIMARY KEY AUTOINCREMENT, \
-                         DataSetName TEXT, \
-                         NodeId INTEGER,\
-                         {sub_query}\
-                         )"
-        )
-
-        # Create FLpipeline table
-        cursor.execute("CREATE TABLE FLpipeline(\
-         id INTEGER PRIMARY KEY AUTOINCREMENT,\
-         name TEXT NOT NULL, \
-         description TEXT NOT NULL,\
-         creation_date TEXT NOT NULL,\
-         results TEXT NOT NULL\
-        ) ")
-
-        # Create test results table
-        cursor.execute("CREATE TABLE testResults(\
-         pipelineId INTEGER,\
-         nodename TEXT NOT NULL, \
-         confusionmatrix TEXT,\
-         accuracy REAL,\
-         sensivity REAL,\
-         ppv REAL,\
-         npv REAL,\
-         f1score REAL,\
-         fpr REAL,\
-         tpr REAL, \
-         PRIMARY KEY (pipelineId, nodename)\
-        ) ")
-
-        # Create FederatedDataset table
-        cursor.execute("CREATE TABLE FedDatasets (\
-         FedId INTEGER PRIMARY KEY AUTOINCREMENT,\
-         FLsetupId INTEGER,\
-         FLpipeId INTEGER,\
-         name TEXT NOT NULL\
-        )")
-
-        # Commit and close the cursor
-        conn.commit()
-        cursor.close()
-        conn.close()
-
-    except sqlite3.Error as e:
-        print(f"Error: {e}")
-
-if __name__ == "__main__":
-    if len(sys.argv) != 2:
-        print("Usage: python script.py <path_to_csv_file>")
-        sys.exit(1)
-    csv_file_path = sys.argv[1]
-    main(csv_file_path)