fedops 0.2.2__tar.gz → 0.12.1__tar.gz

{fedops-0.2.2 → fedops-0.12.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fedops
-Version: 0.2.2
+Version: 0.12.1
 Summary: FL Lifecycle Operations Management Platform
 Home-page: https://github.com/gachon-CCLab/FedOps.git
 Author: Semo Yang

fedops-0.12.1/fedops/client/app.py

@@ -0,0 +1,218 @@
+import logging, json
+import time
+from fastapi import FastAPI, BackgroundTasks
+import asyncio
+import uvicorn
+from datetime import datetime
+
+from . import client_utils
+from . import client_fl
+from . import client_wandb
+from . import client_api
+
+
+class FLClientTask():
+    def __init__(self, cfg, fl_task=None, FL_client_port=None):
+        self.app = FastAPI()
+        self.status = client_utils.FLClientStatus()
+        self.cfg = cfg
+        self.client_port = FL_client_port
+        self.task_id = cfg.task_id
+        self.client_name = cfg.client.name
+        self.dataset_name = cfg.dataset.name
+        self.output_size = cfg.model.output_size
+        self.validation_split = cfg.dataset.validation_split
+        self.wandb_use = cfg.wandb.use
+        self.model_type = cfg.model_type
+        self.model = fl_task["model"]
+        self.model_name = fl_task["model_name"]
+        self.y_label_counter = fl_task["y_label_counter"]
+        
+        logging.info(f'init model_type: {self.model_type}')
+        
+        if self.wandb_use:
+            self.wandb_key = cfg.wandb.key
+            self.wandb_account = cfg.wandb.account
+            self.wandb_project = cfg.wandb.project
+
+        if self.model_type=="Tensorflow":
+            self.x_train = fl_task["x_train"]
+            self.x_test = fl_task["x_test"]
+            self.y_train = fl_task["y_train"]
+            self.y_test = fl_task["y_test"]
+
+        elif self.model_type == "Pytorch":
+            self.train_loader = fl_task["train_loader"]
+            self.val_loader = fl_task["val_loader"]
+            self.test_loader = fl_task["test_loader"]
+            self.criterion = fl_task["criterion"]
+            self.optimizer = fl_task["optimizer"]
+            self.train_torch = fl_task["train_torch"]
+            self.test_torch = fl_task["test_torch"]
+                    
+
+    async def fl_client_start(self):
+        logging.info('FL learning ready')
+
+        logging.info(f'fl_task_id: {self.task_id}')
+        logging.info(f'dataset: {self.dataset_name}')
+        logging.info(f'output_size: {self.output_size}')
+        logging.info(f'validation_split: {self.validation_split}')
+        logging.info(f'model_type: {self.model_type}')
+
+        """
+        Before running this code,
+        set wandb api and account in the config.yaml
+        """
+        if self.wandb_use:
+            logging.info(f'wandb_key: {self.wandb_key}')
+            logging.info(f'wandb_account: {self.wandb_account}')
+            # Set the name in the wandb project
+            wandb_name = f"client-v{self.status.next_gl_model}({datetime.now()})"
+            # Login and init wandb project
+            wandb_run = client_wandb.start_wandb(self.wandb_key, self.wandb_project, wandb_name)
+        else:
+            wandb_run=None
+        
+        # Initialize wandb_config, client object
+        wandb_config = {}
+        # client = None
+        
+        try:
+            loop = asyncio.get_event_loop()
+            if self.model_type == "Tensorflow":
+                # Update wandb config
+                wandb_config = {"learning_rate": self.model.optimizer.learning_rate.numpy(),
+                                "optimizer": self.model.optimizer,
+                                "loss_function": self.model.loss,
+                                "dataset": self.dataset_name, "model_architecture": self.model_name}
+
+                client = client_fl.FLClient(model=self.model, x_train=self.x_train, y_train=self.y_train, x_test=self.x_test,
+                                            y_test=self.y_test,
+                                            validation_split=self.validation_split, fl_task_id=self.task_id, client_name=self.status.client_mac,
+                                            fl_round=1, next_gl_model=self.status.next_gl_model, wandb_use=self.wandb_use,
+                                            wandb_run=wandb_run, model_name=self.model_name, model_type=self.model_type)
+
+            elif self.model_type == "Pytorch":
+                wandb_config = {"learning_rate": self.optimizer.param_groups[0]['lr'],
+                                "optimizer": self.optimizer.__class__.__name__,
+                                "loss_function": self.criterion,
+                                "dataset": self.dataset_name, "model_architecture": self.model_name}
+
+                client = client_fl.FLClient(model=self.model, validation_split=self.validation_split, 
+                                            fl_task_id=self.task_id, client_name=self.status.client_name,
+                                            fl_round=1, next_gl_model=self.status.next_gl_model, wandb_use=self.wandb_use,
+                                            wandb_run=wandb_run, model_name=self.model_name, model_type=self.model_type, 
+                                            train_loader=self.train_loader, val_loader=self.val_loader, test_loader=self.test_loader, 
+                                            criterion=self.criterion, optimizer=self.optimizer, 
+                                            train_torch=self.train_torch, test_torch=self.test_torch)
+
+
+            
+            # Check data fl client data status in the wandb
+            label_values = [[i, self.y_label_counter[i]] for i in range(self.output_size)]
+            logging.info(f'label_values: {label_values}')
+
+            # client_start object
+            client_start = client_fl.flower_client_start(self.status.server_IP, client)
+
+            # FL client start time
+            fl_start_time = time.time()
+
+            # Run asynchronously FL client
+            await loop.run_in_executor(None, client_start)
+
+            logging.info('fl learning finished')
+
+            # FL client end time
+            fl_end_time = time.time() - fl_start_time
+            
+            
+            if self.wandb_use:
+                wandb_run.config.update(wandb_config, allow_val_change=True)
+                client_wandb.data_status_wandb(wandb_run, label_values)
+                # Wandb log(Client round end time)
+                wandb_run.log({"operation_time": fl_end_time, "next_gl_model_v": self.status.next_gl_model},step=self.status.next_gl_model)
+                # close wandb
+                wandb_run.finish()
+                
+                # Get client system result from wandb and send it to client_performance pod
+                client_wandb.client_system_wandb(self.task_id, self.status.client_name, self.status.next_gl_model,
+                                             wandb_name, self.wandb_account)
+
+            client_all_time_result = {"fl_task_id": self.task_id, "client_mac": self.status.client_name,
+                                      "operation_time": fl_end_time,
+                                      "next_gl_model_v": self.status.next_gl_model}
+            json_result = json.dumps(client_all_time_result)
+            logging.info(f'client_operation_time - {json_result}')
+
+            # Send client_time_result to client_performance pod
+            client_api.ClientServerAPI(self.task_id).put_client_time_result(json_result)
+
+            # Delete client object
+            del client
+
+            # Complete Client training
+            self.status.client_start = await client_utils.notify_fin()
+            logging.info('FL Client Learning Finish')
+
+        except Exception as e:
+            logging.info('[E][PC0002] learning', e)
+            self.status.client_fail = True
+            self.status.client_start = await client_utils.notify_fail()
+            raise e
+
+    def start(self):
+        # Configure routes, endpoints, and other FastAPI-related logic here
+        # Example:
+        @self.app.get('/online')
+        async def get_info():
+            return self.status
+
+        # asynchronously start client
+        @self.app.post("/start")
+        async def client_start_trigger(background_tasks: BackgroundTasks):
+
+            # client_manager address
+            # client_res = client_api.ClientMangerAPI().get_info()
+
+            # # # latest global model version
+            # latest_gl_model_v = client_res.json()['GL_Model_V']
+
+            # # next global model version
+            # self.status.next_gl_model = latest_gl_model_v + 1
+            self.status.next_gl_model = 1
+
+            logging.info('bulid model')
+
+            logging.info('FL start')
+            self.status.client_start = True
+
+            # self.status.FL_server_IP = manager_data.server_ip
+            self.status.task_id = self.task_id
+            self.status.client_name = client_utils.get_mac_address()
+            self.status.client_name = self.client_name
+
+            # get the FL server IP
+            # self.status.FL_server_IP = client_api.ClientServerAPI(self.task_id).get_port()
+            self.status.server_IP = "0.0.0.0:8080"
+
+            # start FL Client
+            background_tasks.add_task(self.fl_client_start)
+
+            return self.status
+
+        try:
+            # create client api => to connect client manager
+            uvicorn.run(self.app, host='0.0.0.0', port=self.client_port)
+
+        except Exception as e:
+            # Handle any exceptions that occur during the execution
+            logging.error(f'An error occurred during execution: {e}')
+
+        finally:
+            # FL client out
+            client_api.ClientMangerAPI().get_client_out()
+            logging.info(f'{self.status.client_name}-client close')
+
+

{fedops-0.2.2 → fedops-0.12.1}/fedops/client/client_api.py

@@ -45,7 +45,7 @@ class ClientMangerAPI():
 class ClientServerAPI():
     def __init__(self, task_id):
         self.task_id = task_id
-        self.ccl_address = 'ccljhub.gachon.ac.kr'
+        self.ccl_address = 'ccl.gachon.ac.kr'
         self.server_manager_port = '40019'
         self.client_performance_port = '40015'
 

fedops-0.12.1/fedops/client/client_fl.py

@@ -0,0 +1,242 @@
+from collections import OrderedDict
+import json, logging
+import flwr as fl
+import time
+from functools import partial
+from . import client_api
+from . import client_utils
+
+# set log format
+handlers_list = [logging.StreamHandler()]
+
+# if os.environ["MONITORING"] == '1':
+#     handlers_list.append(logging.FileHandler('./fedops/fl_client.log'))
+# else:
+#     pass
+
+logging.basicConfig(level=logging.DEBUG, format="%(asctime)s [%(levelname)8.8s] %(message)s",
+                    handlers=handlers_list)
+
+logger = logging.getLogger(__name__)
+
+class FLClient(fl.client.NumPyClient):
+
+    def __init__(self, model, validation_split, fl_task_id, client_name, fl_round,next_gl_model, wandb_use, 
+                 wandb_run=None, model_name=None, model_type=None, x_train=None, y_train=None, x_test=None, y_test=None, 
+                 train_loader=None, val_loader=None, test_loader=None, criterion=None, optimizer=None, train_torch=None, test_torch=None):
+        self.model_type = model_type
+        self.model = model
+        self.validation_split = validation_split
+        self.fl_task_id = fl_task_id
+        self.client_name = client_name
+        self.fl_round = fl_round
+        self.next_gl_model = next_gl_model
+        self.model_name = model_name
+        self.wandb_use = wandb_use
+        
+        if self.wandb_use:
+            self.wandb_run = wandb_run
+        
+        if self.model_type == "Tensorflow": 
+            self.x_train, self.y_train = x_train, y_train
+            self.x_test, self.y_test = x_test, y_test
+        
+        elif self.model_type == "Pytorch":
+            self.train_loader = train_loader
+            self.val_loader = val_loader
+            self.test_loader = test_loader
+            self.criterion = criterion
+            self.optimizer = optimizer
+            self.train_torch = train_torch
+            self.test_torch = test_torch
+
+
+    def set_parameters(self, parameters):
+        import torch
+        """Loads a efficientnet model and replaces it parameters with the ones
+        given."""
+        keys = [k for k in self.model.state_dict().keys() if "bn" not in k] # Excluding parameters of BN layers
+        params_dict = zip(keys, parameters)
+        state_dict = OrderedDict({k: torch.tensor(v) for k, v in params_dict})
+        # self.model.load_state_dict(state_dict, strict=True)
+        self.model.load_state_dict(state_dict, strict=False)
+    
+    def get_parameters(self):
+        """Get parameters of the local model."""
+        if self.model_type == "Tensorflow":
+            raise Exception("Not implemented (server-side parameter initialization)")
+        
+        elif self.model_type == "Pytorch":
+            return [
+                val.cpu().numpy() for name, val in self.model.state_dict().items() 
+                if "bn" not in name # Excluding parameters of BN layers
+            ]
+
+    def get_properties(self, config):
+        """Get properties of client."""
+        raise Exception("Not implemented")
+
+    def fit(self, parameters, config):
+        """Train parameters on the locally held training set."""
+
+        print(f"config: {config}")
+        # Get hyperparameters for this round
+        batch_size: int = config["batch_size"]
+        epochs: int = config["local_epochs"]
+        num_rounds: int = config["num_rounds"]
+
+        if self.wandb_use:
+            # add wandb config
+            self.wandb_run.config.update({"batch_size": batch_size, "epochs": epochs, "num_rounds": num_rounds}, allow_val_change=True)
+
+        # start round time
+        round_start_time = time.time()
+
+        # model path for saving local model
+        model_path = f'./local_model/{self.fl_task_id}/{self.model_name}_local_model_V{self.next_gl_model}'
+
+        # Initialize results
+        results = {}
+        
+        # Training Tensorflow
+        if self.model_type == "Tensorflow":
+            # Update local model parameters
+            self.model.set_weights(parameters)
+            
+            # Train the model using hyperparameters from config
+            history = self.model.fit(
+                self.x_train,
+                self.y_train,
+                batch_size,
+                epochs,
+                validation_split=self.validation_split,
+            )
+
+            train_loss = history.history["loss"][len(history.history["loss"])-1]
+            train_accuracy = history.history["accuracy"][len(history.history["accuracy"])-1]
+            results = {
+                "train_loss": train_loss,
+                "train_accuracy": train_accuracy,
+                "val_loss": history.history["val_loss"][len(history.history["val_loss"])-1],
+                "val_accuracy": history.history["val_accuracy"][len(history.history["val_accuracy"])-1],
+            }
+
+            # Return updated model parameters
+            parameters_prime = self.model.get_weights()
+            num_examples_train = len(self.x_train)
+
+            
+            # save local model
+            self.model.save(model_path+'.h5')
+            
+
+        # Training Torch
+        elif self.model_type == "Pytorch":
+            # Update local model parameters
+            self.set_parameters(parameters)
+            
+            # results = client_utils.torch_train(self.model, self.train_dataset, self.criterion, self.optimizer,
+            #                                     self.validation_split, epochs, batch_size)
+            trained_model = self.train_torch(self.model, self.train_loader, self.criterion, self.optimizer, 
+                                            epochs)
+            
+            train_loss, train_accuracy = self.test_torch(trained_model, self.train_loader, self.criterion)
+            val_loss, val_accuracy = self.test_torch(trained_model, self.val_loader, self.criterion)
+            
+            results = {"train_loss": train_loss,
+                    "train_accuracy": train_accuracy,
+                    "val_loss": val_loss,
+                    "val_accuracy": val_accuracy,
+                    }
+            
+            # Return updated model parameters
+            parameters_prime = client_utils.get_model_params(self.model)
+            num_examples_train = len(self.train_loader)
+            
+            # Save model weights
+            import torch
+            torch.save(self.model.state_dict(), model_path+'.pth')
+        else:
+            raise ValueError("Unsupported model_type")
+
+
+        # end round time
+        round_end_time = time.time() - round_start_time
+
+        if self.wandb_use:
+            # wandb train log
+            self.wandb_run.log({"train_loss": results["train_loss"], "round": self.fl_round}, step=self.fl_round) # train_loss
+            self.wandb_run.log({"train_accuracy": results["train_accuracy"], "round": self.fl_round}, step=self.fl_round)  # train_accuracy
+            self.wandb_run.log({"train_time": round_end_time, "round": self.fl_round}, step=self.fl_round)  # train time
+
+            # wandb test log
+            self.wandb_run.log({"val_loss": results["val_loss"], "round": self.fl_round}, step=self.fl_round) # test_loss
+            self.wandb_run.log({"val_accuracy": results["val_accuracy"], "round": self.fl_round}, step=self.fl_round)  # test_accuracy
+
+
+        # Training: model performance by round
+        train_result = {"fl_task_id": self.fl_task_id, "client_name": self.client_name, "round": self.fl_round,
+                        "train_loss": results["train_loss"], "train_accuracy": results["train_accuracy"],
+                        "test_loss": results["val_loss"], "test_accuracy": results["val_accuracy"], 
+                        "train_time": round_end_time, "next_gl_model_v": self.next_gl_model}
+        json_result = json.dumps(train_result)
+        logger.info(f'train_performance - {json_result}')
+
+        # send train_result to client_performance pod
+        client_api.ClientServerAPI(self.fl_task_id).put_train_result(json_result)
+
+        return parameters_prime, num_examples_train, results
+
+
+    def evaluate(self, parameters, config):
+        """Evaluate parameters on the locally held test set."""
+
+        # Get config values
+        batch_size: int = config["batch_size"]
+
+        # Initialize test_loss, test_accuracy
+        test_loss = 0.0
+        test_accuracy = 0.0
+        
+        if self.model_type == "Tensorflow":
+            # Update local model with global parameters
+            self.model.set_weights(parameters)
+            
+            # Evaluate global model parameters on the local test data and return results
+            test_loss, test_accuracy = self.model.evaluate(x=self.x_test, y=self.y_test, batch_size=batch_size)
+
+            num_examples_test = len(self.x_test)
+            
+        elif self.model_type == "Pytorch":            
+            # Update local model parameters
+            self.set_parameters(parameters)
+            
+            # Evaluate global model parameters on the local test data and return results
+            test_loss, test_accuracy = self.test_torch(self.model, self.test_loader, self.criterion)
+            num_examples_test = len(self.test_loader)
+        else:
+            raise ValueError("Unsupported model_type")
+
+        if self.wandb_use:
+            # wandb log
+            self.wandb_run.log({"test_loss": test_loss, "round": self.fl_round}, step=self.fl_round)  # loss
+            self.wandb_run.log({"test_acc": test_accuracy, "round": self.fl_round}, step=self.fl_round)  # acc
+
+        # Test: model performance by round
+        test_result = {"fl_task_id": self.fl_task_id, "client_name": self.client_name, "round": self.fl_round,
+                       "test_loss": test_loss, "test_acc": test_accuracy, "next_gl_model_v": self.next_gl_model}
+        json_result = json.dumps(test_result)
+        logger.info(f'test - {json_result}')
+
+        # send test_result to client_performance pod
+        client_api.ClientServerAPI(self.fl_task_id).put_test_result(json_result)
+
+        # increase next round
+        self.fl_round += 1
+
+        return test_loss, num_examples_test, {"accuracy": test_accuracy}
+
+
+def flower_client_start(FL_server_IP, client):
+    client_start = partial(fl.client.start_numpy_client, server_address=FL_server_IP, client=client)
+    return client_start