MEDfl 2.0.10.dev8__tar.gz → 2.0.10.dev10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/client.py +16 -4
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/holdout.py +371 -2
- medfl-2.0.10.dev10/MEDfl/rw/identity.py +12 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/xgboost/client.py +17 -6
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/PKG-INFO +1 -1
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/medfl.egg-info/PKG-INFO +1 -1
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/medfl.egg-info/SOURCES.txt +1 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/setup.py +1 -1
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/LICENSE +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/__init__.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/client.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/dynamicModal.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/federated_dataset.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/flpipeline.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/model.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/params.yaml +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/params_optimiser.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/plot.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/server.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/shap/__init__.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/shap/aggregation.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/shap/config.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/shap/explainer.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/shap/xgboost_explainer.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/strategy.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/utils.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/xgboost/__init__.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/xgboost/client.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/xgboost/config.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/xgboost/data.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/xgboost/metrics.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/xgboost/serialization.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/xgboost/server.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/xgboost/strategy.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/NetManager/__init__.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/NetManager/database_connector.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/NetManager/dataset.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/NetManager/flsetup.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/NetManager/net_helper.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/NetManager/net_manager_queries.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/NetManager/network.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/NetManager/node.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/__init__.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/__init__.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/model.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/server.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/strategy.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/xgboost/__init__.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/xgboost/config.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/xgboost/data.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/xgboost/metrics.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/xgboost/serialization.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/xgboost/strategy.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/scripts/__init__.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/scripts/base.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/scripts/create_db.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/README.md +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/alembic/__init__.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/alembic/env.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/medfl.egg-info/dependency_links.txt +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/medfl.egg-info/requires.txt +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/medfl.egg-info/top_level.txt +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/setup.cfg +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/tests/test_dataset.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/tests/test_federated_dataset.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/tests/test_flpipeline.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/tests/test_flsetup.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/tests/test_network.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/tests/test_node.py +0 -0
- {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/tests/test_rw_holdout.py +0 -0
|
@@ -13,6 +13,7 @@ from sklearn.model_selection import train_test_split
|
|
|
13
13
|
from MEDfl.rw.model import Net # votre définition de modèle
|
|
14
14
|
from MEDfl.LearningManager.shap import LocalPyTorchSHAPExplainer
|
|
15
15
|
from MEDfl.rw.holdout import run_local_holdout
|
|
16
|
+
from MEDfl.rw.identity import client_name
|
|
16
17
|
import socket
|
|
17
18
|
import platform
|
|
18
19
|
import psutil
|
|
@@ -426,7 +427,7 @@ class FlowerClient(fl.client.NumPyClient):
|
|
|
426
427
|
|
|
427
428
|
# Convert NumPy arrays to lists before JSON serialization.
|
|
428
429
|
serializable_result = {
|
|
429
|
-
"client_id":
|
|
430
|
+
"client_id": client_name(),
|
|
430
431
|
"feature_names": feature_names,
|
|
431
432
|
"available_samples": int(len(loader.dataset)),
|
|
432
433
|
"sample_count": explained_samples,
|
|
@@ -502,7 +503,7 @@ class FlowerClient(fl.client.NumPyClient):
|
|
|
502
503
|
auc = float("nan")
|
|
503
504
|
acc = accuracy_score(y_true, binary_preds)
|
|
504
505
|
|
|
505
|
-
hostname =
|
|
506
|
+
hostname = client_name()
|
|
506
507
|
os_type = platform.system()
|
|
507
508
|
metrics = {
|
|
508
509
|
"hostname": hostname,
|
|
@@ -608,8 +609,9 @@ class FlowerClient(fl.client.NumPyClient):
|
|
|
608
609
|
features=self.effective_features,
|
|
609
610
|
target=self.effective_target,
|
|
610
611
|
predict=self._predict_probabilities,
|
|
611
|
-
client_id=
|
|
612
|
+
client_id=client_name(),
|
|
612
613
|
drop_missing_features=True,
|
|
614
|
+
validation=self._validation_predictions,
|
|
613
615
|
)
|
|
614
616
|
)
|
|
615
617
|
|
|
@@ -624,8 +626,18 @@ class FlowerClient(fl.client.NumPyClient):
|
|
|
624
626
|
logits = self.model(torch.tensor(X, dtype=torch.float32))
|
|
625
627
|
return torch.sigmoid(logits.reshape(-1)).cpu().numpy()
|
|
626
628
|
|
|
629
|
+
def _validation_predictions(self):
|
|
630
|
+
"""(y_true, y_prob) of the validation split, for the optimized holdout threshold."""
|
|
631
|
+
if self.val_loader is None:
|
|
632
|
+
return np.empty((0,)), np.empty((0,))
|
|
633
|
+
y_parts, prob_parts = [], []
|
|
634
|
+
for X_batch, y_batch in self.val_loader:
|
|
635
|
+
y_parts.append(y_batch.cpu().numpy().reshape(-1))
|
|
636
|
+
prob_parts.append(self._predict_probabilities(X_batch.cpu().numpy()))
|
|
637
|
+
return np.concatenate(y_parts), np.concatenate(prob_parts)
|
|
638
|
+
|
|
627
639
|
def get_properties(self, config):
|
|
628
|
-
hostname =
|
|
640
|
+
hostname = client_name()
|
|
629
641
|
os_type = platform.system()
|
|
630
642
|
|
|
631
643
|
if self._initialized:
|
|
@@ -17,6 +17,14 @@ histogram bin width: two scores falling in the same bin count as a tie.
|
|
|
17
17
|
The holdout goes through the same preprocessing as the client training
|
|
18
18
|
data: the server features in the server order, the raw values (the
|
|
19
19
|
real-world clients do not impute or scale), and a binary 0/1 target.
|
|
20
|
+
|
|
21
|
+
Optimized threshold (threshold_source="optimized"): in the same final
|
|
22
|
+
round, every client also sends the two score histograms of its validation
|
|
23
|
+
split (never used for training). The server sums them and picks the
|
|
24
|
+
threshold among the histogram bin edges (Youden's J or a target
|
|
25
|
+
sensitivity). At a bin edge, the histograms give exact confusion matrices,
|
|
26
|
+
so the holdout results are then recomputed from the holdout histograms
|
|
27
|
+
at that threshold. The holdout histograms are never used to choose it.
|
|
20
28
|
"""
|
|
21
29
|
|
|
22
30
|
from __future__ import annotations
|
|
@@ -37,6 +45,10 @@ HOLDOUT_MISSING = "missing" # no holdout file on this client
|
|
|
37
45
|
HOLDOUT_FAILED = "failed"
|
|
38
46
|
HOLDOUT_UNSUPPORTED = "unsupported_task"
|
|
39
47
|
|
|
48
|
+
THRESHOLD_SOURCES = ("manual", "optimized")
|
|
49
|
+
# criterion key -> label shown in the results
|
|
50
|
+
THRESHOLD_CRITERIA = {"youden": "Youden's J", "target_sensitivity": "Target sensitivity"}
|
|
51
|
+
|
|
40
52
|
|
|
41
53
|
@dataclass
|
|
42
54
|
class HoldoutConfig:
|
|
@@ -57,6 +69,16 @@ class HoldoutConfig:
|
|
|
57
69
|
|
|
58
70
|
include_client_results: bool = True
|
|
59
71
|
|
|
72
|
+
# "manual": the threshold above. "optimized": chosen on the clients'
|
|
73
|
+
# validation splits, at a histogram bin edge (resolution 1 / num_bins)
|
|
74
|
+
threshold_source: str = "manual"
|
|
75
|
+
|
|
76
|
+
# "youden" or "target_sensitivity" (optimized threshold only)
|
|
77
|
+
threshold_criterion: str = "youden"
|
|
78
|
+
|
|
79
|
+
# minimal pooled validation sensitivity of the target_sensitivity criterion
|
|
80
|
+
target_sensitivity: float = 0.9
|
|
81
|
+
|
|
60
82
|
def validate(self) -> None:
|
|
61
83
|
if not isinstance(self.enabled, bool):
|
|
62
84
|
raise TypeError("Holdout enabled must be a boolean")
|
|
@@ -70,6 +92,19 @@ class HoldoutConfig:
|
|
|
70
92
|
if int(self.minimum_samples) <= 0:
|
|
71
93
|
raise ValueError("Holdout minimum_samples must be greater than zero")
|
|
72
94
|
|
|
95
|
+
if self.threshold_source not in THRESHOLD_SOURCES:
|
|
96
|
+
raise ValueError(
|
|
97
|
+
f"Holdout threshold_source must be one of {THRESHOLD_SOURCES}"
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
if self.threshold_criterion not in THRESHOLD_CRITERIA:
|
|
101
|
+
raise ValueError(
|
|
102
|
+
f"Holdout threshold_criterion must be one of {tuple(THRESHOLD_CRITERIA)}"
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
if not 0.0 < float(self.target_sensitivity) <= 1.0:
|
|
106
|
+
raise ValueError("Holdout target_sensitivity must be in (0, 1]")
|
|
107
|
+
|
|
73
108
|
def to_evaluate_config(self) -> Dict[str, Any]:
|
|
74
109
|
"""
|
|
75
110
|
Keys added to the evaluate config of the final round.
|
|
@@ -80,6 +115,8 @@ class HoldoutConfig:
|
|
|
80
115
|
"holdout_threshold": float(self.threshold),
|
|
81
116
|
"holdout_num_bins": int(self.num_bins),
|
|
82
117
|
"holdout_minimum_samples": int(self.minimum_samples),
|
|
118
|
+
# the clients also send their validation histograms
|
|
119
|
+
"threshold_optimization_enabled": self.threshold_source == "optimized",
|
|
83
120
|
}
|
|
84
121
|
|
|
85
122
|
|
|
@@ -242,6 +279,117 @@ def local_holdout_summary(
|
|
|
242
279
|
}
|
|
243
280
|
|
|
244
281
|
|
|
282
|
+
def local_validation_summary(
|
|
283
|
+
y_true: np.ndarray,
|
|
284
|
+
y_prob: np.ndarray,
|
|
285
|
+
num_bins: int,
|
|
286
|
+
minimum_samples: int,
|
|
287
|
+
client_id: str,
|
|
288
|
+
) -> Dict[str, Any]:
|
|
289
|
+
"""
|
|
290
|
+
Summary of one client's validation split for the optimized threshold,
|
|
291
|
+
safe to send to the server: counts and the two score histograms (same
|
|
292
|
+
bins as the holdout). Below minimum_samples rows, only the counts are sent.
|
|
293
|
+
"""
|
|
294
|
+
y_true = np.asarray(y_true, dtype=float).reshape(-1)
|
|
295
|
+
y_prob = np.asarray(y_prob, dtype=float).reshape(-1)
|
|
296
|
+
|
|
297
|
+
# e.g. missing feature values in the neural network input
|
|
298
|
+
predicted = np.isfinite(y_prob)
|
|
299
|
+
y_true, y_prob = y_true[predicted], y_prob[predicted]
|
|
300
|
+
|
|
301
|
+
classes = set(np.unique(y_true).tolist())
|
|
302
|
+
if not classes.issubset({0.0, 1.0}):
|
|
303
|
+
raise ValueError(
|
|
304
|
+
f"The validation target must be binary (0/1), found {len(classes)} "
|
|
305
|
+
f"values: {sorted(classes)[:10]}"
|
|
306
|
+
)
|
|
307
|
+
|
|
308
|
+
rows = int(len(y_true))
|
|
309
|
+
positives = int(y_true.sum())
|
|
310
|
+
summary = {
|
|
311
|
+
"client_id": client_id,
|
|
312
|
+
"dataset": {
|
|
313
|
+
"rows": rows,
|
|
314
|
+
"positives": positives,
|
|
315
|
+
"negatives": rows - positives,
|
|
316
|
+
"rows_without_prediction": int((~predicted).sum()),
|
|
317
|
+
},
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
if rows == 0:
|
|
321
|
+
return {
|
|
322
|
+
**summary,
|
|
323
|
+
"status": HOLDOUT_MISSING,
|
|
324
|
+
"reason": "No validation rows (validation fraction of 0?)",
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
if rows < int(minimum_samples):
|
|
328
|
+
return {
|
|
329
|
+
**summary,
|
|
330
|
+
"status": HOLDOUT_SKIPPED,
|
|
331
|
+
"reason": (
|
|
332
|
+
f"{rows} validation row(s), fewer than the minimum of "
|
|
333
|
+
f"{int(minimum_samples)}"
|
|
334
|
+
),
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
y_prob = np.clip(y_prob, 0.0, 1.0)
|
|
338
|
+
edges = _bin_edges(num_bins)
|
|
339
|
+
return {
|
|
340
|
+
**summary,
|
|
341
|
+
"status": HOLDOUT_COMPLETED,
|
|
342
|
+
"num_bins": int(num_bins),
|
|
343
|
+
"pos_hist": np.histogram(y_prob[y_true == 1], bins=edges)[0].astype(int).tolist(),
|
|
344
|
+
"neg_hist": np.histogram(y_prob[y_true == 0], bins=edges)[0].astype(int).tolist(),
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def run_local_validation(
|
|
349
|
+
config: Dict[str, Any],
|
|
350
|
+
validation: Optional[Callable[[], Tuple[np.ndarray, np.ndarray]]],
|
|
351
|
+
client_id: str,
|
|
352
|
+
) -> Dict[str, Any]:
|
|
353
|
+
"""
|
|
354
|
+
Entries to add to the Flower evaluate metrics for the optimized
|
|
355
|
+
threshold: threshold_validation_status, then threshold_validation
|
|
356
|
+
(JSON) or threshold_validation_error.
|
|
357
|
+
|
|
358
|
+
validation returns (y_true, y_prob) of the client's validation split
|
|
359
|
+
with the final global model.
|
|
360
|
+
"""
|
|
361
|
+
if validation is None:
|
|
362
|
+
return {
|
|
363
|
+
"threshold_validation_status": HOLDOUT_MISSING,
|
|
364
|
+
"threshold_validation_error": "This client cannot send validation data",
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
try:
|
|
368
|
+
y_true, y_prob = validation()
|
|
369
|
+
summary = local_validation_summary(
|
|
370
|
+
y_true=y_true,
|
|
371
|
+
y_prob=y_prob,
|
|
372
|
+
num_bins=int(config.get("holdout_num_bins", 100)),
|
|
373
|
+
minimum_samples=int(config.get("holdout_minimum_samples", 10)),
|
|
374
|
+
client_id=client_id,
|
|
375
|
+
)
|
|
376
|
+
print(
|
|
377
|
+
f"[Client/Holdout] Validation for the threshold: {summary['status']} | "
|
|
378
|
+
f"client={client_id} | rows={summary['dataset']['rows']}",
|
|
379
|
+
flush=True,
|
|
380
|
+
)
|
|
381
|
+
return {
|
|
382
|
+
"threshold_validation_status": summary["status"],
|
|
383
|
+
"threshold_validation": json.dumps(summary, allow_nan=False),
|
|
384
|
+
}
|
|
385
|
+
except Exception as exc:
|
|
386
|
+
print(f"[Client/Holdout] Validation for the threshold failed: {exc}", flush=True)
|
|
387
|
+
return {
|
|
388
|
+
"threshold_validation_status": HOLDOUT_FAILED,
|
|
389
|
+
"threshold_validation_error": str(exc),
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
|
|
245
393
|
def run_local_holdout(
|
|
246
394
|
config: Dict[str, Any],
|
|
247
395
|
holdout_path: Optional[str],
|
|
@@ -250,6 +398,7 @@ def run_local_holdout(
|
|
|
250
398
|
predict: Callable[[np.ndarray], np.ndarray],
|
|
251
399
|
client_id: str,
|
|
252
400
|
drop_missing_features: bool,
|
|
401
|
+
validation: Optional[Callable[[], Tuple[np.ndarray, np.ndarray]]] = None,
|
|
253
402
|
) -> Dict[str, Any]:
|
|
254
403
|
"""
|
|
255
404
|
Run the holdout test on a client and return the entries to add to the
|
|
@@ -258,7 +407,29 @@ def run_local_holdout(
|
|
|
258
407
|
|
|
259
408
|
predict maps the holdout features to positive-class probabilities with
|
|
260
409
|
the final global model.
|
|
410
|
+
|
|
411
|
+
When the server optimizes the threshold, the validation summary is
|
|
412
|
+
added too (see run_local_validation), even without a holdout file:
|
|
413
|
+
the client validation data still helps choose the threshold.
|
|
261
414
|
"""
|
|
415
|
+
entries = {}
|
|
416
|
+
if bool(config.get("threshold_optimization_enabled", False)):
|
|
417
|
+
entries = run_local_validation(config, validation, client_id)
|
|
418
|
+
return {
|
|
419
|
+
**entries,
|
|
420
|
+
**_holdout_entries(config, holdout_path, features, target, predict, client_id, drop_missing_features),
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def _holdout_entries(
|
|
425
|
+
config: Dict[str, Any],
|
|
426
|
+
holdout_path: Optional[str],
|
|
427
|
+
features: List[str],
|
|
428
|
+
target: str,
|
|
429
|
+
predict: Callable[[np.ndarray], np.ndarray],
|
|
430
|
+
client_id: str,
|
|
431
|
+
drop_missing_features: bool,
|
|
432
|
+
) -> Dict[str, Any]:
|
|
262
433
|
if not holdout_path or not os.path.exists(holdout_path):
|
|
263
434
|
print(f"[Client/Holdout] No holdout file on this client ({holdout_path})", flush=True)
|
|
264
435
|
return {
|
|
@@ -328,6 +499,178 @@ def _client_name(client_proxy) -> str:
|
|
|
328
499
|
return str(getattr(client_proxy, "cid", client_proxy))
|
|
329
500
|
|
|
330
501
|
|
|
502
|
+
def histogram_confusion(pos_hist: np.ndarray, neg_hist: np.ndarray, edge: int) -> Dict[str, int]:
|
|
503
|
+
"""
|
|
504
|
+
Exact confusion matrix at the threshold bins_edges[edge]: the scores at
|
|
505
|
+
or above it are the bins edge, edge + 1, ... (np.histogram bins are
|
|
506
|
+
[left, right) and the last one is closed).
|
|
507
|
+
"""
|
|
508
|
+
pos_hist, neg_hist = np.asarray(pos_hist), np.asarray(neg_hist)
|
|
509
|
+
return {
|
|
510
|
+
"TP": int(pos_hist[edge:].sum()),
|
|
511
|
+
"FP": int(neg_hist[edge:].sum()),
|
|
512
|
+
"FN": int(pos_hist[:edge].sum()),
|
|
513
|
+
"TN": int(neg_hist[:edge].sum()),
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def select_threshold_from_histograms(
|
|
518
|
+
pos_hist: np.ndarray,
|
|
519
|
+
neg_hist: np.ndarray,
|
|
520
|
+
criterion: str,
|
|
521
|
+
target_sensitivity: Optional[float] = None,
|
|
522
|
+
) -> Tuple[int, float]:
|
|
523
|
+
"""
|
|
524
|
+
Choose the threshold among the inner bin edges of the score histograms
|
|
525
|
+
and return (edge index, threshold).
|
|
526
|
+
|
|
527
|
+
youden: maximizes sensitivity + specificity - 1, the middle edge of the
|
|
528
|
+
best ones when several tie.
|
|
529
|
+
target_sensitivity: the highest edge whose sensitivity reaches the
|
|
530
|
+
target, so the best specificity under that constraint.
|
|
531
|
+
"""
|
|
532
|
+
pos_hist, neg_hist = np.asarray(pos_hist), np.asarray(neg_hist)
|
|
533
|
+
num_bins = len(pos_hist)
|
|
534
|
+
positives, negatives = int(pos_hist.sum()), int(neg_hist.sum())
|
|
535
|
+
if positives == 0 or negatives == 0:
|
|
536
|
+
raise ValueError(
|
|
537
|
+
f"The validation data has a single class ({positives} positive, "
|
|
538
|
+
f"{negatives} negative): the threshold cannot be optimized"
|
|
539
|
+
)
|
|
540
|
+
|
|
541
|
+
# candidate thresholds, strictly between 0 and 1
|
|
542
|
+
edges = np.arange(1, num_bins)
|
|
543
|
+
# scores in bin k and above
|
|
544
|
+
positives_above = np.cumsum(pos_hist[::-1])[::-1]
|
|
545
|
+
negatives_above = np.cumsum(neg_hist[::-1])[::-1]
|
|
546
|
+
sensitivity = positives_above[edges] / positives
|
|
547
|
+
specificity = 1 - negatives_above[edges] / negatives
|
|
548
|
+
|
|
549
|
+
if criterion == "youden":
|
|
550
|
+
youden = sensitivity + specificity - 1
|
|
551
|
+
ties = np.flatnonzero(youden >= youden.max() - 1e-12)
|
|
552
|
+
edge = int(edges[ties[len(ties) // 2]])
|
|
553
|
+
elif criterion == "target_sensitivity":
|
|
554
|
+
target = float(target_sensitivity)
|
|
555
|
+
if not 0.0 < target <= 1.0:
|
|
556
|
+
raise ValueError(f"The target sensitivity must be in (0, 1] (got {target})")
|
|
557
|
+
# sensitivity decreases with the edge
|
|
558
|
+
reached = np.flatnonzero(sensitivity >= target - 1e-12)
|
|
559
|
+
if not len(reached):
|
|
560
|
+
raise ValueError(
|
|
561
|
+
f"No threshold reaches a validation sensitivity of {target}: "
|
|
562
|
+
f"{int(pos_hist[0])} of the {positives} positive case(s) score below "
|
|
563
|
+
f"the lowest candidate threshold {1 / num_bins:g}. Increase the number "
|
|
564
|
+
f"of bins or lower the target sensitivity"
|
|
565
|
+
)
|
|
566
|
+
edge = int(edges[reached.max()])
|
|
567
|
+
else:
|
|
568
|
+
raise ValueError(f"Unknown threshold criterion '{criterion}'")
|
|
569
|
+
|
|
570
|
+
return edge, float(_bin_edges(num_bins)[edge])
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def optimize_federated_threshold(results, holdout_config: HoldoutConfig) -> Dict[str, Any]:
|
|
574
|
+
"""
|
|
575
|
+
Choose the threshold on the pooled validation histograms of the final
|
|
576
|
+
round. Returns the selection with status "completed" (threshold, edge,
|
|
577
|
+
validation metrics) or "failed" (error), and the excluded clients.
|
|
578
|
+
"""
|
|
579
|
+
completed, skipped, missing, failed = [], [], [], []
|
|
580
|
+
|
|
581
|
+
for client_proxy, eval_res in results:
|
|
582
|
+
metrics = eval_res.metrics or {}
|
|
583
|
+
status = metrics.get("threshold_validation_status")
|
|
584
|
+
name = str(metrics.get("holdout_client_id") or _client_name(client_proxy))
|
|
585
|
+
|
|
586
|
+
if status is None:
|
|
587
|
+
missing.append({
|
|
588
|
+
"client_id": name,
|
|
589
|
+
"error": "No validation data sent (the client MEDfl version may not support the optimized threshold)",
|
|
590
|
+
})
|
|
591
|
+
continue
|
|
592
|
+
|
|
593
|
+
summary = None
|
|
594
|
+
if metrics.get("threshold_validation"):
|
|
595
|
+
try:
|
|
596
|
+
summary = json.loads(metrics["threshold_validation"])
|
|
597
|
+
except Exception as exc:
|
|
598
|
+
failed.append({"client_id": name, "error": f"Invalid validation JSON: {exc}"})
|
|
599
|
+
continue
|
|
600
|
+
|
|
601
|
+
if status == HOLDOUT_COMPLETED and summary:
|
|
602
|
+
if int(summary.get("num_bins", -1)) != int(holdout_config.num_bins):
|
|
603
|
+
failed.append({
|
|
604
|
+
"client_id": summary.get("client_id", name),
|
|
605
|
+
"error": f"Used {summary.get('num_bins')} histogram bins instead of {holdout_config.num_bins}",
|
|
606
|
+
})
|
|
607
|
+
continue
|
|
608
|
+
completed.append(summary)
|
|
609
|
+
elif status == HOLDOUT_SKIPPED and summary:
|
|
610
|
+
skipped.append({
|
|
611
|
+
"client_id": summary.get("client_id", name),
|
|
612
|
+
"rows": summary["dataset"]["rows"],
|
|
613
|
+
"reason": summary.get("reason"),
|
|
614
|
+
})
|
|
615
|
+
elif status == HOLDOUT_MISSING:
|
|
616
|
+
missing.append({
|
|
617
|
+
"client_id": name,
|
|
618
|
+
"error": (summary or {}).get("reason") or metrics.get("threshold_validation_error"),
|
|
619
|
+
})
|
|
620
|
+
else:
|
|
621
|
+
failed.append({
|
|
622
|
+
"client_id": name,
|
|
623
|
+
"error": metrics.get("threshold_validation_error", "No validation result returned"),
|
|
624
|
+
})
|
|
625
|
+
|
|
626
|
+
criterion = holdout_config.threshold_criterion
|
|
627
|
+
selection: Dict[str, Any] = {
|
|
628
|
+
"criterion": criterion,
|
|
629
|
+
"criterion_label": THRESHOLD_CRITERIA.get(criterion, criterion),
|
|
630
|
+
"target_sensitivity": (
|
|
631
|
+
float(holdout_config.target_sensitivity) if criterion == "target_sensitivity" else None
|
|
632
|
+
),
|
|
633
|
+
"method": f"pooled {int(holdout_config.num_bins)}-bin validation score histograms",
|
|
634
|
+
"num_bins": int(holdout_config.num_bins),
|
|
635
|
+
# the threshold is a bin edge
|
|
636
|
+
"resolution": 1.0 / int(holdout_config.num_bins),
|
|
637
|
+
"skipped_clients": skipped,
|
|
638
|
+
"missing_clients": missing,
|
|
639
|
+
"failed_clients": failed,
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
if not completed:
|
|
643
|
+
return {**selection, "status": "failed", "error": "No client returned validation data"}
|
|
644
|
+
|
|
645
|
+
pos_hist = np.sum([summary["pos_hist"] for summary in completed], axis=0)
|
|
646
|
+
neg_hist = np.sum([summary["neg_hist"] for summary in completed], axis=0)
|
|
647
|
+
positives, negatives = int(pos_hist.sum()), int(neg_hist.sum())
|
|
648
|
+
selection["validation"] = {
|
|
649
|
+
"clients": len(completed),
|
|
650
|
+
"rows": positives + negatives,
|
|
651
|
+
"positives": positives,
|
|
652
|
+
"negatives": negatives,
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
try:
|
|
656
|
+
edge, threshold = select_threshold_from_histograms(
|
|
657
|
+
pos_hist, neg_hist, criterion, holdout_config.target_sensitivity
|
|
658
|
+
)
|
|
659
|
+
except ValueError as exc:
|
|
660
|
+
return {**selection, "status": "failed", "error": str(exc)}
|
|
661
|
+
|
|
662
|
+
confusion = histogram_confusion(pos_hist, neg_hist, edge)
|
|
663
|
+
auc, _ = pooled_roc_auc(pos_hist, neg_hist)
|
|
664
|
+
return {
|
|
665
|
+
**selection,
|
|
666
|
+
"status": "completed",
|
|
667
|
+
"threshold": threshold,
|
|
668
|
+
"edge": edge,
|
|
669
|
+
"metrics": {"auc": auc, **confusion_metrics(confusion)},
|
|
670
|
+
"confusion_matrix": confusion,
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
|
|
331
674
|
def aggregate_federated_holdout(
|
|
332
675
|
results,
|
|
333
676
|
failures,
|
|
@@ -390,9 +733,10 @@ def aggregate_federated_holdout(
|
|
|
390
733
|
else:
|
|
391
734
|
failed.append({"client_id": _client_name(failure[0]), "error": "Evaluation failed"})
|
|
392
735
|
|
|
736
|
+
threshold = float(holdout_config.threshold)
|
|
393
737
|
result: Dict[str, Any] = {
|
|
394
738
|
"backend": backend,
|
|
395
|
-
"threshold":
|
|
739
|
+
"threshold": threshold,
|
|
396
740
|
"num_bins": int(holdout_config.num_bins),
|
|
397
741
|
"minimum_samples": int(holdout_config.minimum_samples),
|
|
398
742
|
"participating_clients": len(completed),
|
|
@@ -401,6 +745,31 @@ def aggregate_federated_holdout(
|
|
|
401
745
|
"failed_clients": failed,
|
|
402
746
|
}
|
|
403
747
|
|
|
748
|
+
if holdout_config.threshold_source == "optimized":
|
|
749
|
+
# chosen on the validation histograms only, the holdout ones are not read here
|
|
750
|
+
selection = optimize_federated_threshold(results, holdout_config)
|
|
751
|
+
result["threshold_selection"] = selection
|
|
752
|
+
if selection["status"] != "completed":
|
|
753
|
+
# no fallback to the manual threshold
|
|
754
|
+
return {
|
|
755
|
+
**result,
|
|
756
|
+
"status": "failed",
|
|
757
|
+
"error": f"Threshold optimization failed: {selection['error']}",
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
threshold = selection["threshold"]
|
|
761
|
+
result["threshold"] = threshold
|
|
762
|
+
# the client metrics were computed at the manual threshold: recompute them exactly at the bin edge
|
|
763
|
+
for summary in completed:
|
|
764
|
+
summary["threshold"] = threshold
|
|
765
|
+
summary["confusion_matrix"] = histogram_confusion(
|
|
766
|
+
summary["pos_hist"], summary["neg_hist"], selection["edge"]
|
|
767
|
+
)
|
|
768
|
+
summary["metrics"] = {
|
|
769
|
+
"auc": summary["metrics"].get("auc"),
|
|
770
|
+
**confusion_metrics(summary["confusion_matrix"]),
|
|
771
|
+
}
|
|
772
|
+
|
|
404
773
|
if not completed:
|
|
405
774
|
return {
|
|
406
775
|
**result,
|
|
@@ -442,7 +811,7 @@ def aggregate_federated_holdout(
|
|
|
442
811
|
sum(summary["dataset"]["rows_without_target"] for summary in completed)
|
|
443
812
|
),
|
|
444
813
|
},
|
|
445
|
-
"threshold":
|
|
814
|
+
"threshold": threshold,
|
|
446
815
|
"metrics": {"auc": auc, **confusion_metrics(confusion)},
|
|
447
816
|
"confusion_matrix": confusion,
|
|
448
817
|
"roc_curve": roc,
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Name a real-world client reports to the server (properties, metrics, SHAP and holdout results).
|
|
3
|
+
|
|
4
|
+
The MEDfl agent ID (MEDFL_AGENT_ID, inherited from the agent that starts the client) when it is set,
|
|
5
|
+
so several agents on the same machine are told apart. Else the machine hostname, as before.
|
|
6
|
+
"""
|
|
7
|
+
import os
|
|
8
|
+
import socket
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def client_name() -> str:
|
|
12
|
+
return (os.getenv("MEDFL_AGENT_ID") or "").strip() or socket.gethostname()
|
|
@@ -3,7 +3,7 @@ import json
|
|
|
3
3
|
import socket
|
|
4
4
|
import platform
|
|
5
5
|
import shutil
|
|
6
|
-
from typing import Dict, Any, Optional
|
|
6
|
+
from typing import Dict, Any, Optional, Tuple
|
|
7
7
|
|
|
8
8
|
import flwr as fl
|
|
9
9
|
import numpy as np
|
|
@@ -31,6 +31,7 @@ try:
|
|
|
31
31
|
except ImportError:
|
|
32
32
|
GPUtil = None
|
|
33
33
|
|
|
34
|
+
from MEDfl.rw.identity import client_name
|
|
34
35
|
from MEDfl.rw.xgboost.data import XGBoostDataModule
|
|
35
36
|
from MEDfl.rw.xgboost.metrics import evaluate_predictions
|
|
36
37
|
from MEDfl.rw.xgboost.serialization import (
|
|
@@ -374,7 +375,7 @@ class XGBoostFlowerClient(fl.client.Client):
|
|
|
374
375
|
)
|
|
375
376
|
|
|
376
377
|
return {
|
|
377
|
-
"client_id":
|
|
378
|
+
"client_id": client_name(),
|
|
378
379
|
"feature_names": feature_names,
|
|
379
380
|
"available_samples": available_samples,
|
|
380
381
|
"sample_count": explained_samples,
|
|
@@ -457,7 +458,7 @@ class XGBoostFlowerClient(fl.client.Client):
|
|
|
457
458
|
metrics = {
|
|
458
459
|
"backend": "xgboost",
|
|
459
460
|
"task": self.task,
|
|
460
|
-
"hostname":
|
|
461
|
+
"hostname": client_name(),
|
|
461
462
|
"os_type": platform.system(),
|
|
462
463
|
"num_train": int(self.metadata.get("num_train", 0)),
|
|
463
464
|
"features": ",".join(self.metadata.get("features", [])),
|
|
@@ -574,7 +575,7 @@ class XGBoostFlowerClient(fl.client.Client):
|
|
|
574
575
|
if bool(config.get("holdout_enabled", False)):
|
|
575
576
|
if self.task != "binary":
|
|
576
577
|
metrics["holdout_status"] = HOLDOUT_UNSUPPORTED
|
|
577
|
-
metrics["holdout_client_id"] =
|
|
578
|
+
metrics["holdout_client_id"] = client_name()
|
|
578
579
|
metrics["holdout_error"] = (
|
|
579
580
|
"The holdout test supports binary classification "
|
|
580
581
|
f"only (task is '{self.task}')"
|
|
@@ -587,8 +588,9 @@ class XGBoostFlowerClient(fl.client.Client):
|
|
|
587
588
|
features=self.data_module.effective_features,
|
|
588
589
|
target=self.data_module.effective_target,
|
|
589
590
|
predict=self._predict_probabilities,
|
|
590
|
-
client_id=
|
|
591
|
+
client_id=client_name(),
|
|
591
592
|
drop_missing_features=False,
|
|
593
|
+
validation=self._validation_predictions,
|
|
592
594
|
)
|
|
593
595
|
)
|
|
594
596
|
|
|
@@ -607,12 +609,21 @@ class XGBoostFlowerClient(fl.client.Client):
|
|
|
607
609
|
)
|
|
608
610
|
return np.asarray(self.booster.predict(matrix), dtype=float)
|
|
609
611
|
|
|
612
|
+
def _validation_predictions(self) -> Tuple[np.ndarray, np.ndarray]:
|
|
613
|
+
"""(y_true, y_prob) of the validation split, for the optimized holdout threshold."""
|
|
614
|
+
if self.dval is None or int(self.metadata.get("num_val", 0)) == 0:
|
|
615
|
+
return np.empty((0,)), np.empty((0,))
|
|
616
|
+
return (
|
|
617
|
+
np.asarray(self.dval.get_label(), dtype=float),
|
|
618
|
+
np.asarray(self.booster.predict(self.dval), dtype=float),
|
|
619
|
+
)
|
|
620
|
+
|
|
610
621
|
def get_properties(self, ins: GetPropertiesIns) -> GetPropertiesRes:
|
|
611
622
|
"""
|
|
612
623
|
Return machine and dataset properties.
|
|
613
624
|
"""
|
|
614
625
|
|
|
615
|
-
hostname =
|
|
626
|
+
hostname = client_name()
|
|
616
627
|
os_type = platform.system()
|
|
617
628
|
|
|
618
629
|
df = self.data_module.df
|
|
@@ -14,7 +14,7 @@ with open('requirements.txt') as f:
|
|
|
14
14
|
|
|
15
15
|
setup(
|
|
16
16
|
name="medfl",
|
|
17
|
-
version="2.0.10.
|
|
17
|
+
version="2.0.10.dev10",
|
|
18
18
|
author="MEDomics consortium",
|
|
19
19
|
author_email="medomics.info@gmail.com",
|
|
20
20
|
description="Python Open-source package for simulating federated learning and differential privacy",
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|