MEDfl 2.0.10.dev6__tar.gz → 2.0.10.dev7__tar.gz

{medfl-2.0.10.dev6 → medfl-2.0.10.dev7}/MEDfl/rw/server.py

@@ -66,12 +66,17 @@ class FederatedServer:
         # the actual Flower ServerConfig.
         self.strategy_wrapper.total_rounds = int(num_rounds)
 
-        # Validate the PyTorch SHAP configuration before the server starts.
+        # Validate SHAP against the active real-world backend.
         if self.shap_config.enabled:
             feature_columns = [
                 column.strip()
                 for column in (
-                    self.strategy_wrapper._features or ""
+                    getattr(
+                        self.strategy_wrapper,
+                        "_features",
+                        "",
+                    )
+                    or ""
                 ).split(",")
                 if column.strip()
             ]
@@ -79,13 +84,34 @@ class FederatedServer:
             if not feature_columns:
                 raise ValueError(
                     "Federated SHAP requires the Strategy 'features' "
-                    "configuration so the neural-network input size "
-                    "can be validated."
+                    "configuration so the model input size can be "
+                    "validated."
                 )
 
+            backend = (
+                "xgboost"
+                if getattr(
+                    self.strategy_wrapper,
+                    "name",
+                    "",
+                ) == "FedXgbBagging"
+                else "pytorch"
+            )
+
+            task = (
+                getattr(
+                    self.strategy_wrapper,
+                    "task",
+                    None,
+                )
+                if backend == "xgboost"
+                else None
+            )
+
             self.shap_config.validate_backend(
                 input_size=len(feature_columns),
-                backend="pytorch",
+                backend=backend,
+                task=task,
             )
 
         self.strategy_wrapper.create_strategy()
@@ -196,4 +222,4 @@ class TrackingClientManager(fl.server.client_manager.SimpleClientManager):
         #     hostname = f"Error: {e}"
 
         print(f"✅ Client connected - CID: {client.cid}")
-        self.server.connected_clients.append(client.cid)
+        self.server.connected_clients.append(client.cid)

{medfl-2.0.10.dev6 → medfl-2.0.10.dev7}/MEDfl/rw/xgboost/client.py

@@ -10,6 +10,8 @@ import numpy as np
 import pandas as pd
 import psutil
 import xgboost as xgb
+import torch
+from torch.utils.data import DataLoader, TensorDataset
 from flwr.common import (
     Code,
     EvaluateIns,
@@ -35,6 +37,9 @@ from MEDfl.rw.xgboost.serialization import (
     booster_to_parameters,
     parameters_to_booster,
 )
+from MEDfl.LearningManager.shap import (
+    LocalXGBoostSHAPExplainer,
+)
 
 
 class XGBoostFlowerClient(fl.client.Client):
@@ -144,6 +149,250 @@ class XGBoostFlowerClient(fl.client.Client):
             f"test={self.metadata.get('num_test')}"
         )
 
+    # ------------------------------------------------------------------
+    # Federated SHAP helpers
+    # ------------------------------------------------------------------
+
+    def _get_shap_dmatrix(self, data_split: str) -> xgb.DMatrix:
+        """
+        Return the local DMatrix selected for the post-training SHAP phase.
+        """
+        split = str(data_split).lower()
+
+        if split == "train":
+            matrix = self.dtrain
+        elif split == "validation":
+            matrix = self.dval
+        elif split == "test":
+            matrix = self.dtest
+        else:
+            raise ValueError(
+                f"Unsupported SHAP data split '{data_split}'. "
+                "Expected train, validation, or test."
+            )
+
+        if matrix is None:
+            raise ValueError(
+                f"The selected SHAP split '{split}' is not available "
+                "on this XGBoost client."
+            )
+
+        if int(matrix.num_row()) <= 0:
+            raise ValueError(
+                f"The selected SHAP split '{split}' is empty "
+                "on this XGBoost client."
+            )
+
+        return matrix
+
+    @staticmethod
+    def _dmatrix_to_shap_loader(
+        matrix: xgb.DMatrix,
+        batch_size: int = 256,
+    ) -> DataLoader:
+        """
+        Convert a local XGBoost DMatrix into the DataLoader interface expected
+        by LocalXGBoostSHAPExplainer.
+
+        The conversion is local only. No samples are transmitted.
+        """
+        if matrix is None:
+            raise ValueError("Cannot create a SHAP loader from None.")
+
+        try:
+            features = matrix.get_data()
+        except AttributeError as exc:
+            raise RuntimeError(
+                "This XGBoost version does not expose DMatrix.get_data(). "
+                "Upgrade XGBoost or adapt XGBoostDataModule to retain the "
+                "local NumPy feature splits for SHAP."
+            ) from exc
+
+        # DMatrix.get_data() commonly returns a scipy sparse matrix.
+        if hasattr(features, "toarray"):
+            features = features.toarray()
+
+        features = np.asarray(features, dtype=np.float32)
+
+        if features.ndim != 2:
+            raise ValueError(
+                "SHAP expects 2-D tabular XGBoost features. "
+                f"Received shape {features.shape}."
+            )
+
+        labels = np.asarray(
+            matrix.get_label(),
+            dtype=np.float32,
+        )
+
+        x_tensor = torch.tensor(
+            features,
+            dtype=torch.float32,
+        )
+        y_tensor = torch.tensor(
+            labels,
+            dtype=torch.float32,
+        )
+
+        return DataLoader(
+            TensorDataset(x_tensor, y_tensor),
+            batch_size=max(1, min(int(batch_size), len(x_tensor))),
+            shuffle=False,
+        )
+
+    def _calculate_local_shap(
+        self,
+        config: Dict[str, Any],
+    ) -> Dict[str, Any]:
+        """
+        Explain the final federated XGBoost booster on this client's selected
+        local data partition.
+
+        Only additive SHAP summaries are serialized back to the server.
+        """
+        if not self._initialized:
+            self._initialize_from_config(config)
+
+        if self.booster is None:
+            raise RuntimeError(
+                "Cannot calculate XGBoost SHAP without a booster."
+            )
+
+        explainer_name = str(
+            config.get("shap_explainer", "tree")
+        ).lower()
+
+        if explainer_name != "tree":
+            raise ValueError(
+                "Real-world XGBoost SHAP requires explainer='tree'."
+            )
+
+        data_split = str(
+            config.get("shap_data_split", "validation")
+        )
+        background_size = int(
+            config.get("shap_background_size", 100)
+        )
+        explanation_size = int(
+            config.get("shap_explanation_size", 500)
+        )
+        random_seed = int(
+            config.get("shap_random_seed", 42)
+        )
+        minimum_samples = int(
+            config.get("shap_minimum_samples", 10)
+        )
+        model_output = str(
+            config.get("shap_model_output", "raw")
+        )
+
+        clipping_raw = float(
+            config.get("shap_clipping_value", -1.0)
+        )
+        clipping_value = (
+            None if clipping_raw <= 0 else clipping_raw
+        )
+
+        feature_names_raw = str(
+            config.get("shap_feature_names_json", "[]")
+        )
+
+        try:
+            configured_feature_names = json.loads(
+                feature_names_raw
+            )
+        except Exception:
+            configured_feature_names = []
+
+        effective_features = list(
+            self.metadata.get("features", [])
+        )
+
+        feature_names = (
+            list(configured_feature_names)
+            if configured_feature_names
+            else effective_features
+        )
+
+        if len(feature_names) != int(
+            self.metadata.get("num_features", 0)
+        ):
+            raise ValueError(
+                "SHAP feature names do not match the XGBoost input size. "
+                f"Expected {self.metadata.get('num_features', 0)}, "
+                f"received {len(feature_names)}."
+            )
+
+        matrix = self._get_shap_dmatrix(data_split)
+
+        available_samples = int(matrix.num_row())
+
+        if available_samples < minimum_samples:
+            raise ValueError(
+                f"Only {available_samples} samples are available in "
+                f"the '{data_split}' split, but "
+                f"minimum_samples={minimum_samples}."
+            )
+
+        loader = self._dmatrix_to_shap_loader(
+            matrix=matrix,
+            batch_size=min(256, available_samples),
+        )
+
+        local_explainer = LocalXGBoostSHAPExplainer(
+            background_size=background_size,
+            explanation_size=explanation_size,
+            random_seed=random_seed,
+            clipping_value=clipping_value,
+            model_output=model_output,
+        )
+
+        result = local_explainer.calculate(
+            booster=self.booster,
+            loader=loader,
+        )
+
+        explained_samples = int(
+            result["sample_count"]
+        )
+
+        if explained_samples < minimum_samples:
+            raise ValueError(
+                f"Only {explained_samples} samples were explained, "
+                f"but minimum_samples={minimum_samples}."
+            )
+
+        return {
+            "client_id": socket.gethostname(),
+            "feature_names": feature_names,
+            "available_samples": available_samples,
+            "sample_count": explained_samples,
+            "abs_sum": np.asarray(
+                result["abs_sum"],
+                dtype=np.float64,
+            ).tolist(),
+            "signed_sum": np.asarray(
+                result["signed_sum"],
+                dtype=np.float64,
+            ).tolist(),
+            "square_sum": np.asarray(
+                result["square_sum"],
+                dtype=np.float64,
+            ).tolist(),
+            "positive_count": np.asarray(
+                result["positive_count"],
+                dtype=np.int64,
+            ).tolist(),
+            "negative_count": np.asarray(
+                result["negative_count"],
+                dtype=np.int64,
+            ).tolist(),
+            "zero_count": np.asarray(
+                result["zero_count"],
+                dtype=np.int64,
+            ).tolist(),
+        }
+
     def get_parameters(self, ins: GetParametersIns) -> GetParametersRes:
         """
         Return current booster parameters.
@@ -265,6 +514,48 @@ class XGBoostFlowerClient(fl.client.Client):
         else:
             loss = float(eval_metrics.get("mlogloss", 0.0))
 
+        # --------------------------------------------------------------
+        # Final-round federated SHAP.
+        #
+        # ins.parameters contains the server-aggregated booster for this
+        # evaluation. Therefore all participating clients explain the
+        # same final global XGBoost model.
+        # --------------------------------------------------------------
+        if bool(config.get("shap_enabled", False)):
+            print(
+                "[XGBoostClient/SHAP] Starting local SHAP on "
+                "the final global booster...",
+                flush=True,
+            )
+
+            try:
+                local_shap_result = (
+                    self._calculate_local_shap(config)
+                )
+
+                metrics["shap_status"] = "completed"
+                metrics["shap_result"] = json.dumps(
+                    local_shap_result,
+                    allow_nan=False,
+                )
+
+                print(
+                    "[XGBoostClient/SHAP] Local SHAP completed | "
+                    f"client={local_shap_result['client_id']} | "
+                    f"samples={local_shap_result['sample_count']}",
+                    flush=True,
+                )
+
+            except Exception as exc:
+                metrics["shap_status"] = "failed"
+                metrics["shap_error"] = str(exc)
+
+                print(
+                    "[XGBoostClient/SHAP] Local SHAP failed: "
+                    f"{exc}",
+                    flush=True,
+                )
+
         return EvaluateRes(
             status=Status(code=Code.OK, message="Success"),
             loss=loss,

{medfl-2.0.10.dev6 → medfl-2.0.10.dev7}/MEDfl/rw/xgboost/strategy.py

@@ -8,6 +8,7 @@ import flwr as fl
 from flwr.common import (
     Parameters,
     FitRes,
+    EvaluateIns,
     parameters_to_ndarrays,
 )
 from flwr.common import GetPropertiesIns
@@ -19,6 +20,10 @@ from MEDfl.rw.xgboost.serialization import (
     save_booster,
     XGBOOST_TENSOR_TYPE,
 )
+from MEDfl.LearningManager.shap import (
+    SHAPConfig,
+    aggregate_federated_shap,
+)
 
 
 class XGBoostStrategy:
@@ -65,6 +70,7 @@ class XGBoostStrategy:
         savingPath: str = "",
         saveOnRounds: int = 1,
         total_rounds: int = 3,
+        shap_config: Optional[SHAPConfig] = None,
     ):
         self.name = "FedXgbBagging"
         self.mode = mode
@@ -109,6 +115,20 @@ class XGBoostStrategy:
         self.saveOnRounds = saveOnRounds
         self.total_rounds = total_rounds
 
+        self.shap_config = (
+            shap_config
+            if shap_config is not None
+            else SHAPConfig(enabled=False)
+        )
+        self.federated_shap_result = {
+            "status": (
+                "pending"
+                if self.shap_config.enabled
+                else "disabled"
+            )
+        }
+        self.local_shap_results = []
+
         self.strategy_object = None
         self.current_booster: Optional[xgb.Booster] = None
         self.current_num_trees: int = 0
@@ -137,7 +157,11 @@ class XGBoostStrategy:
             }
 
         def evaluate_config_fn(server_round: int) -> Dict[str, Any]:
-            return {
+            is_final_round = (
+                int(server_round) == int(self.total_rounds)
+            )
+
+            cfg = {
                 "backend": "xgboost",
                 "mode": self.mode,
                 "task": self.task,
@@ -147,10 +171,56 @@ class XGBoostStrategy:
                 "val_fraction": float(self._val_fraction),
                 "test_fraction": float(self._test_fraction),
                 "id_col": self.id_col,
-                "local_num_boost_round": int(self.local_num_boost_round),
+                "local_num_boost_round": int(
+                    self.local_num_boost_round
+                ),
                 "xgb_params": json.dumps(self.xgb_params),
+                "shap_enabled": bool(
+                    self.shap_config.enabled
+                    and is_final_round
+                ),
             }
 
+            if cfg["shap_enabled"]:
+                cfg.update(
+                    {
+                        "shap_explainer": (
+                            self.shap_config.explainer
+                        ),
+                        "shap_data_split": (
+                            self.shap_config.data_split
+                        ),
+                        "shap_background_size": int(
+                            self.shap_config.background_size
+                        ),
+                        "shap_explanation_size": int(
+                            self.shap_config.explanation_size
+                        ),
+                        "shap_random_seed": int(
+                            self.shap_config.random_seed
+                        ),
+                        "shap_minimum_samples": int(
+                            self.shap_config.minimum_samples
+                        ),
+                        "shap_clipping_value": (
+                            -1.0
+                            if self.shap_config.clipping_value
+                            is None
+                            else float(
+                                self.shap_config.clipping_value
+                            )
+                        ),
+                        "shap_feature_names_json": json.dumps(
+                            self.shap_config.feature_names or []
+                        ),
+                        "shap_model_output": (
+                            self.shap_config.model_output
+                        ),
+                    }
+                )
+
+            return cfg
+
         strategy = fl.server.strategy.FedAvg(
             fraction_fit=self.fraction_fit,
             fraction_evaluate=self.fraction_evaluate,
@@ -168,6 +238,7 @@ class XGBoostStrategy:
         )
 
         self._wrap_configure_fit(strategy)
+        self._wrap_configure_evaluate(strategy, evaluate_config_fn)
         self._wrap_aggregate_fit(strategy)
         self._wrap_aggregate_evaluate(strategy)
 
@@ -259,6 +330,63 @@ class XGBoostStrategy:
 
         strategy.configure_fit = wrapped_configure_fit
 
+    def _wrap_configure_evaluate(
+        self,
+        strategy,
+        evaluate_config_fn,
+    ) -> None:
+        """
+        On the final round, request evaluation/SHAP from every currently
+        connected client so the federated SHAP result represents all
+        available institutions.
+        """
+        original_configure_evaluate = (
+            strategy.configure_evaluate
+        )
+
+        def wrapped_configure_evaluate(
+            server_round,
+            parameters,
+            client_manager,
+        ):
+            is_final_shap_round = (
+                self.shap_config.enabled
+                and int(server_round)
+                == int(self.total_rounds)
+            )
+
+            if not is_final_shap_round:
+                return original_configure_evaluate(
+                    server_round=server_round,
+                    parameters=parameters,
+                    client_manager=client_manager,
+                )
+
+            evaluate_ins = EvaluateIns(
+                parameters=parameters,
+                config=evaluate_config_fn(server_round),
+            )
+
+            available_clients = list(
+                client_manager.all().values()
+            )
+
+            print(
+                "[Server/SHAP] XGBoost final round: "
+                "requesting local SHAP from "
+                f"{len(available_clients)} connected client(s).",
+                flush=True,
+            )
+
+            return [
+                (client, evaluate_ins)
+                for client in available_clients
+            ]
+
+        strategy.configure_evaluate = (
+            wrapped_configure_evaluate
+        )
+
     def _wrap_aggregate_fit(self, strategy) -> None:
         """
         Wrap aggregate_fit for XGBoost.
@@ -396,13 +524,23 @@ class XGBoostStrategy:
 
     def _wrap_aggregate_evaluate(self, strategy) -> None:
         """
-        Wrap aggregate_evaluate for logging.
+        Wrap aggregate_evaluate for logging and final federated SHAP
+        aggregation.
         """
 
-        original_aggregate_evaluate = strategy.aggregate_evaluate
+        original_aggregate_evaluate = (
+            strategy.aggregate_evaluate
+        )
 
-        def wrapped_aggregate_evaluate(server_round, results, failures):
-            print(f"\n[Server/XGBoost] 📊 Round {server_round} - Client Evaluation Metrics:")
+        def wrapped_aggregate_evaluate(
+            server_round,
+            results,
+            failures,
+        ):
+            print(
+                f"\n[Server/XGBoost] 📊 Round {server_round} - "
+                "Client Evaluation Metrics:"
+            )
 
             for client, eval_res in results:
                 print(
@@ -410,14 +548,156 @@ class XGBoostStrategy:
                     f"Client:{client.cid}: {eval_res.metrics}"
                 )
 
-            loss, metrics = original_aggregate_evaluate(server_round, results, failures)
+            loss, metrics = original_aggregate_evaluate(
+                server_round,
+                results,
+                failures,
+            )
+
+            print(
+                f"[Server/XGBoost] ✅ Round {server_round} - "
+                "Aggregated Evaluation:"
+            )
+            print(
+                f"    Loss: {loss}, Metrics: {metrics}\n"
+            )
+
+            is_final_round = (
+                int(server_round) == int(self.total_rounds)
+            )
+
+            if self.shap_config.enabled and is_final_round:
+                local_results = []
+                shap_failures = []
 
-            print(f"[Server/XGBoost] ✅ Round {server_round} - Aggregated Evaluation:")
-            print(f"    Loss: {loss}, Metrics: {metrics}\n")
+                for client_proxy, eval_res in results:
+                    client_metrics = eval_res.metrics or {}
+
+                    if (
+                        client_metrics.get("shap_status")
+                        == "completed"
+                        and client_metrics.get("shap_result")
+                    ):
+                        try:
+                            local_result = json.loads(
+                                client_metrics["shap_result"]
+                            )
+                            local_results.append(
+                                local_result
+                            )
+                        except Exception as exc:
+                            shap_failures.append(
+                                {
+                                    "client_id": client_proxy.cid,
+                                    "error": (
+                                        "Invalid SHAP JSON: "
+                                        f"{exc}"
+                                    ),
+                                }
+                            )
+                    else:
+                        shap_failures.append(
+                            {
+                                "client_id": client_proxy.cid,
+                                "error": client_metrics.get(
+                                    "shap_error",
+                                    "No SHAP result returned",
+                                ),
+                            }
+                        )
+
+                self.local_shap_results = local_results
+
+                if local_results:
+                    try:
+                        federated_result = (
+                            aggregate_federated_shap(
+                                client_results=local_results,
+                                include_client_results=(
+                                    self.shap_config
+                                    .include_client_results
+                                ),
+                            )
+                        )
+
+                        federated_result["backend"] = (
+                            "xgboost"
+                        )
+                        federated_result["explainer"] = (
+                            self.shap_config.explainer
+                        )
+                        federated_result["data_split"] = (
+                            self.shap_config.data_split
+                        )
+                        federated_result["background_size"] = (
+                            self.shap_config.background_size
+                        )
+                        federated_result[
+                            "requested_explanation_size"
+                        ] = (
+                            self.shap_config.explanation_size
+                        )
+                        federated_result["model_output"] = (
+                            self.shap_config.model_output
+                        )
+
+                        if shap_failures:
+                            federated_result[
+                                "failed_clients"
+                            ] = shap_failures
+
+                        self.federated_shap_result = (
+                            federated_result
+                        )
+
+                        print(
+                            "[Server/SHAP] Federated SHAP "
+                            "completed:"
+                        )
+                        print(
+                            json.dumps(
+                                self.federated_shap_result,
+                                indent=2,
+                            ),
+                            flush=True,
+                        )
+
+                    except Exception as exc:
+                        self.federated_shap_result = {
+                            "status": "failed",
+                            "backend": "xgboost",
+                            "error": str(exc),
+                            "failed_clients": shap_failures,
+                        }
+
+                        print(
+                            "[Server/SHAP] XGBoost federated "
+                            f"SHAP aggregation failed: {exc}",
+                            flush=True,
+                        )
+
+                else:
+                    self.federated_shap_result = {
+                        "status": "failed",
+                        "backend": "xgboost",
+                        "error": (
+                            "SHAP was enabled but no XGBoost "
+                            "client returned a valid SHAP result."
+                        ),
+                        "failed_clients": shap_failures,
+                    }
+
+                    print(
+                        "[Server/SHAP] No valid XGBoost local "
+                        "SHAP results were returned.",
+                        flush=True,
+                    )
 
             return loss, metrics
 
-        strategy.aggregate_evaluate = wrapped_aggregate_evaluate
+        strategy.aggregate_evaluate = (
+            wrapped_aggregate_evaluate
+        )
 
     def _save_if_needed(self, server_round: int, parameters: Parameters) -> None:
         """

{medfl-2.0.10.dev6 → medfl-2.0.10.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: medfl
-Version: 2.0.10.dev6
+Version: 2.0.10.dev7
 Summary: Python Open-source package for simulating federated learning and differential privacy
 Home-page: https://github.com/MEDomics-UdeS/MEDfl
 Author: MEDomics consortium

{medfl-2.0.10.dev6 → medfl-2.0.10.dev7}/medfl.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: medfl
-Version: 2.0.10.dev6
+Version: 2.0.10.dev7
 Summary: Python Open-source package for simulating federated learning and differential privacy
 Home-page: https://github.com/MEDomics-UdeS/MEDfl
 Author: MEDomics consortium

{medfl-2.0.10.dev6 → medfl-2.0.10.dev7}/setup.py

@@ -14,7 +14,7 @@ with open('requirements.txt') as f:
 
 setup(
     name="medfl",
-    version="2.0.10.dev6",
+    version="2.0.10.dev7",
     author="MEDomics consortium",
     author_email="medomics.info@gmail.com",
     description="Python Open-source package for simulating federated learning and differential privacy",