MEDfl 2.0.10.dev8__tar.gz → 2.0.10.dev10__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/client.py +16 -4
  2. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/holdout.py +371 -2
  3. medfl-2.0.10.dev10/MEDfl/rw/identity.py +12 -0
  4. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/xgboost/client.py +17 -6
  5. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/PKG-INFO +1 -1
  6. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/medfl.egg-info/PKG-INFO +1 -1
  7. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/medfl.egg-info/SOURCES.txt +1 -0
  8. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/setup.py +1 -1
  9. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/LICENSE +0 -0
  10. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/__init__.py +0 -0
  11. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/client.py +0 -0
  12. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/dynamicModal.py +0 -0
  13. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/federated_dataset.py +0 -0
  14. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/flpipeline.py +0 -0
  15. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/model.py +0 -0
  16. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/params.yaml +0 -0
  17. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/params_optimiser.py +0 -0
  18. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/plot.py +0 -0
  19. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/server.py +0 -0
  20. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/shap/__init__.py +0 -0
  21. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/shap/aggregation.py +0 -0
  22. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/shap/config.py +0 -0
  23. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/shap/explainer.py +0 -0
  24. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/shap/xgboost_explainer.py +0 -0
  25. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/strategy.py +0 -0
  26. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/utils.py +0 -0
  27. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/xgboost/__init__.py +0 -0
  28. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/xgboost/client.py +0 -0
  29. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/xgboost/config.py +0 -0
  30. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/xgboost/data.py +0 -0
  31. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/xgboost/metrics.py +0 -0
  32. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/xgboost/serialization.py +0 -0
  33. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/xgboost/server.py +0 -0
  34. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/LearningManager/xgboost/strategy.py +0 -0
  35. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/NetManager/__init__.py +0 -0
  36. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/NetManager/database_connector.py +0 -0
  37. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/NetManager/dataset.py +0 -0
  38. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/NetManager/flsetup.py +0 -0
  39. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/NetManager/net_helper.py +0 -0
  40. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/NetManager/net_manager_queries.py +0 -0
  41. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/NetManager/network.py +0 -0
  42. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/NetManager/node.py +0 -0
  43. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/__init__.py +0 -0
  44. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/__init__.py +0 -0
  45. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/model.py +0 -0
  46. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/server.py +0 -0
  47. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/strategy.py +0 -0
  48. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/xgboost/__init__.py +0 -0
  49. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/xgboost/config.py +0 -0
  50. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/xgboost/data.py +0 -0
  51. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/xgboost/metrics.py +0 -0
  52. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/xgboost/serialization.py +0 -0
  53. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/rw/xgboost/strategy.py +0 -0
  54. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/scripts/__init__.py +0 -0
  55. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/scripts/base.py +0 -0
  56. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/MEDfl/scripts/create_db.py +0 -0
  57. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/README.md +0 -0
  58. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/alembic/__init__.py +0 -0
  59. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/alembic/env.py +0 -0
  60. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/medfl.egg-info/dependency_links.txt +0 -0
  61. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/medfl.egg-info/requires.txt +0 -0
  62. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/medfl.egg-info/top_level.txt +0 -0
  63. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/setup.cfg +0 -0
  64. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/tests/test_dataset.py +0 -0
  65. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/tests/test_federated_dataset.py +0 -0
  66. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/tests/test_flpipeline.py +0 -0
  67. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/tests/test_flsetup.py +0 -0
  68. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/tests/test_network.py +0 -0
  69. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/tests/test_node.py +0 -0
  70. {medfl-2.0.10.dev8 → medfl-2.0.10.dev10}/tests/test_rw_holdout.py +0 -0
@@ -13,6 +13,7 @@ from sklearn.model_selection import train_test_split
13
13
  from MEDfl.rw.model import Net # votre définition de modèle
14
14
  from MEDfl.LearningManager.shap import LocalPyTorchSHAPExplainer
15
15
  from MEDfl.rw.holdout import run_local_holdout
16
+ from MEDfl.rw.identity import client_name
16
17
  import socket
17
18
  import platform
18
19
  import psutil
@@ -426,7 +427,7 @@ class FlowerClient(fl.client.NumPyClient):
426
427
 
427
428
  # Convert NumPy arrays to lists before JSON serialization.
428
429
  serializable_result = {
429
- "client_id": socket.gethostname(),
430
+ "client_id": client_name(),
430
431
  "feature_names": feature_names,
431
432
  "available_samples": int(len(loader.dataset)),
432
433
  "sample_count": explained_samples,
@@ -502,7 +503,7 @@ class FlowerClient(fl.client.NumPyClient):
502
503
  auc = float("nan")
503
504
  acc = accuracy_score(y_true, binary_preds)
504
505
 
505
- hostname = socket.gethostname()
506
+ hostname = client_name()
506
507
  os_type = platform.system()
507
508
  metrics = {
508
509
  "hostname": hostname,
@@ -608,8 +609,9 @@ class FlowerClient(fl.client.NumPyClient):
608
609
  features=self.effective_features,
609
610
  target=self.effective_target,
610
611
  predict=self._predict_probabilities,
611
- client_id=socket.gethostname(),
612
+ client_id=client_name(),
612
613
  drop_missing_features=True,
614
+ validation=self._validation_predictions,
613
615
  )
614
616
  )
615
617
 
@@ -624,8 +626,18 @@ class FlowerClient(fl.client.NumPyClient):
624
626
  logits = self.model(torch.tensor(X, dtype=torch.float32))
625
627
  return torch.sigmoid(logits.reshape(-1)).cpu().numpy()
626
628
 
629
+ def _validation_predictions(self):
630
+ """(y_true, y_prob) of the validation split, for the optimized holdout threshold."""
631
+ if self.val_loader is None:
632
+ return np.empty((0,)), np.empty((0,))
633
+ y_parts, prob_parts = [], []
634
+ for X_batch, y_batch in self.val_loader:
635
+ y_parts.append(y_batch.cpu().numpy().reshape(-1))
636
+ prob_parts.append(self._predict_probabilities(X_batch.cpu().numpy()))
637
+ return np.concatenate(y_parts), np.concatenate(prob_parts)
638
+
627
639
  def get_properties(self, config):
628
- hostname = socket.gethostname()
640
+ hostname = client_name()
629
641
  os_type = platform.system()
630
642
 
631
643
  if self._initialized:
@@ -17,6 +17,14 @@ histogram bin width: two scores falling in the same bin count as a tie.
17
17
  The holdout goes through the same preprocessing as the client training
18
18
  data: the server features in the server order, the raw values (the
19
19
  real-world clients do not impute or scale), and a binary 0/1 target.
20
+
21
+ Optimized threshold (threshold_source="optimized"): in the same final
22
+ round, every client also sends the two score histograms of its validation
23
+ split (never used for training). The server sums them and picks the
24
+ threshold among the histogram bin edges (Youden's J or a target
25
+ sensitivity). At a bin edge, the histograms give exact confusion matrices,
26
+ so the holdout results are then recomputed from the holdout histograms
27
+ at that threshold. The holdout histograms are never used to choose it.
20
28
  """
21
29
 
22
30
  from __future__ import annotations
@@ -37,6 +45,10 @@ HOLDOUT_MISSING = "missing" # no holdout file on this client
37
45
  HOLDOUT_FAILED = "failed"
38
46
  HOLDOUT_UNSUPPORTED = "unsupported_task"
39
47
 
48
+ THRESHOLD_SOURCES = ("manual", "optimized")
49
+ # criterion key -> label shown in the results
50
+ THRESHOLD_CRITERIA = {"youden": "Youden's J", "target_sensitivity": "Target sensitivity"}
51
+
40
52
 
41
53
  @dataclass
42
54
  class HoldoutConfig:
@@ -57,6 +69,16 @@ class HoldoutConfig:
57
69
 
58
70
  include_client_results: bool = True
59
71
 
72
+ # "manual": the threshold above. "optimized": chosen on the clients'
73
+ # validation splits, at a histogram bin edge (resolution 1 / num_bins)
74
+ threshold_source: str = "manual"
75
+
76
+ # "youden" or "target_sensitivity" (optimized threshold only)
77
+ threshold_criterion: str = "youden"
78
+
79
+ # minimal pooled validation sensitivity of the target_sensitivity criterion
80
+ target_sensitivity: float = 0.9
81
+
60
82
  def validate(self) -> None:
61
83
  if not isinstance(self.enabled, bool):
62
84
  raise TypeError("Holdout enabled must be a boolean")
@@ -70,6 +92,19 @@ class HoldoutConfig:
70
92
  if int(self.minimum_samples) <= 0:
71
93
  raise ValueError("Holdout minimum_samples must be greater than zero")
72
94
 
95
+ if self.threshold_source not in THRESHOLD_SOURCES:
96
+ raise ValueError(
97
+ f"Holdout threshold_source must be one of {THRESHOLD_SOURCES}"
98
+ )
99
+
100
+ if self.threshold_criterion not in THRESHOLD_CRITERIA:
101
+ raise ValueError(
102
+ f"Holdout threshold_criterion must be one of {tuple(THRESHOLD_CRITERIA)}"
103
+ )
104
+
105
+ if not 0.0 < float(self.target_sensitivity) <= 1.0:
106
+ raise ValueError("Holdout target_sensitivity must be in (0, 1]")
107
+
73
108
  def to_evaluate_config(self) -> Dict[str, Any]:
74
109
  """
75
110
  Keys added to the evaluate config of the final round.
@@ -80,6 +115,8 @@ class HoldoutConfig:
80
115
  "holdout_threshold": float(self.threshold),
81
116
  "holdout_num_bins": int(self.num_bins),
82
117
  "holdout_minimum_samples": int(self.minimum_samples),
118
+ # the clients also send their validation histograms
119
+ "threshold_optimization_enabled": self.threshold_source == "optimized",
83
120
  }
84
121
 
85
122
 
@@ -242,6 +279,117 @@ def local_holdout_summary(
242
279
  }
243
280
 
244
281
 
282
+ def local_validation_summary(
283
+ y_true: np.ndarray,
284
+ y_prob: np.ndarray,
285
+ num_bins: int,
286
+ minimum_samples: int,
287
+ client_id: str,
288
+ ) -> Dict[str, Any]:
289
+ """
290
+ Summary of one client's validation split for the optimized threshold,
291
+ safe to send to the server: counts and the two score histograms (same
292
+ bins as the holdout). Below minimum_samples rows, only the counts are sent.
293
+ """
294
+ y_true = np.asarray(y_true, dtype=float).reshape(-1)
295
+ y_prob = np.asarray(y_prob, dtype=float).reshape(-1)
296
+
297
+ # e.g. missing feature values in the neural network input
298
+ predicted = np.isfinite(y_prob)
299
+ y_true, y_prob = y_true[predicted], y_prob[predicted]
300
+
301
+ classes = set(np.unique(y_true).tolist())
302
+ if not classes.issubset({0.0, 1.0}):
303
+ raise ValueError(
304
+ f"The validation target must be binary (0/1), found {len(classes)} "
305
+ f"values: {sorted(classes)[:10]}"
306
+ )
307
+
308
+ rows = int(len(y_true))
309
+ positives = int(y_true.sum())
310
+ summary = {
311
+ "client_id": client_id,
312
+ "dataset": {
313
+ "rows": rows,
314
+ "positives": positives,
315
+ "negatives": rows - positives,
316
+ "rows_without_prediction": int((~predicted).sum()),
317
+ },
318
+ }
319
+
320
+ if rows == 0:
321
+ return {
322
+ **summary,
323
+ "status": HOLDOUT_MISSING,
324
+ "reason": "No validation rows (validation fraction of 0?)",
325
+ }
326
+
327
+ if rows < int(minimum_samples):
328
+ return {
329
+ **summary,
330
+ "status": HOLDOUT_SKIPPED,
331
+ "reason": (
332
+ f"{rows} validation row(s), fewer than the minimum of "
333
+ f"{int(minimum_samples)}"
334
+ ),
335
+ }
336
+
337
+ y_prob = np.clip(y_prob, 0.0, 1.0)
338
+ edges = _bin_edges(num_bins)
339
+ return {
340
+ **summary,
341
+ "status": HOLDOUT_COMPLETED,
342
+ "num_bins": int(num_bins),
343
+ "pos_hist": np.histogram(y_prob[y_true == 1], bins=edges)[0].astype(int).tolist(),
344
+ "neg_hist": np.histogram(y_prob[y_true == 0], bins=edges)[0].astype(int).tolist(),
345
+ }
346
+
347
+
348
+ def run_local_validation(
349
+ config: Dict[str, Any],
350
+ validation: Optional[Callable[[], Tuple[np.ndarray, np.ndarray]]],
351
+ client_id: str,
352
+ ) -> Dict[str, Any]:
353
+ """
354
+ Entries to add to the Flower evaluate metrics for the optimized
355
+ threshold: threshold_validation_status, then threshold_validation
356
+ (JSON) or threshold_validation_error.
357
+
358
+ validation returns (y_true, y_prob) of the client's validation split
359
+ with the final global model.
360
+ """
361
+ if validation is None:
362
+ return {
363
+ "threshold_validation_status": HOLDOUT_MISSING,
364
+ "threshold_validation_error": "This client cannot send validation data",
365
+ }
366
+
367
+ try:
368
+ y_true, y_prob = validation()
369
+ summary = local_validation_summary(
370
+ y_true=y_true,
371
+ y_prob=y_prob,
372
+ num_bins=int(config.get("holdout_num_bins", 100)),
373
+ minimum_samples=int(config.get("holdout_minimum_samples", 10)),
374
+ client_id=client_id,
375
+ )
376
+ print(
377
+ f"[Client/Holdout] Validation for the threshold: {summary['status']} | "
378
+ f"client={client_id} | rows={summary['dataset']['rows']}",
379
+ flush=True,
380
+ )
381
+ return {
382
+ "threshold_validation_status": summary["status"],
383
+ "threshold_validation": json.dumps(summary, allow_nan=False),
384
+ }
385
+ except Exception as exc:
386
+ print(f"[Client/Holdout] Validation for the threshold failed: {exc}", flush=True)
387
+ return {
388
+ "threshold_validation_status": HOLDOUT_FAILED,
389
+ "threshold_validation_error": str(exc),
390
+ }
391
+
392
+
245
393
  def run_local_holdout(
246
394
  config: Dict[str, Any],
247
395
  holdout_path: Optional[str],
@@ -250,6 +398,7 @@ def run_local_holdout(
250
398
  predict: Callable[[np.ndarray], np.ndarray],
251
399
  client_id: str,
252
400
  drop_missing_features: bool,
401
+ validation: Optional[Callable[[], Tuple[np.ndarray, np.ndarray]]] = None,
253
402
  ) -> Dict[str, Any]:
254
403
  """
255
404
  Run the holdout test on a client and return the entries to add to the
@@ -258,7 +407,29 @@ def run_local_holdout(
258
407
 
259
408
  predict maps the holdout features to positive-class probabilities with
260
409
  the final global model.
410
+
411
+ When the server optimizes the threshold, the validation summary is
412
+ added too (see run_local_validation), even without a holdout file:
413
+ the client validation data still helps choose the threshold.
261
414
  """
415
+ entries = {}
416
+ if bool(config.get("threshold_optimization_enabled", False)):
417
+ entries = run_local_validation(config, validation, client_id)
418
+ return {
419
+ **entries,
420
+ **_holdout_entries(config, holdout_path, features, target, predict, client_id, drop_missing_features),
421
+ }
422
+
423
+
424
+ def _holdout_entries(
425
+ config: Dict[str, Any],
426
+ holdout_path: Optional[str],
427
+ features: List[str],
428
+ target: str,
429
+ predict: Callable[[np.ndarray], np.ndarray],
430
+ client_id: str,
431
+ drop_missing_features: bool,
432
+ ) -> Dict[str, Any]:
262
433
  if not holdout_path or not os.path.exists(holdout_path):
263
434
  print(f"[Client/Holdout] No holdout file on this client ({holdout_path})", flush=True)
264
435
  return {
@@ -328,6 +499,178 @@ def _client_name(client_proxy) -> str:
328
499
  return str(getattr(client_proxy, "cid", client_proxy))
329
500
 
330
501
 
502
+ def histogram_confusion(pos_hist: np.ndarray, neg_hist: np.ndarray, edge: int) -> Dict[str, int]:
503
+ """
504
+ Exact confusion matrix at the threshold bins_edges[edge]: the scores at
505
+ or above it are the bins edge, edge + 1, ... (np.histogram bins are
506
+ [left, right) and the last one is closed).
507
+ """
508
+ pos_hist, neg_hist = np.asarray(pos_hist), np.asarray(neg_hist)
509
+ return {
510
+ "TP": int(pos_hist[edge:].sum()),
511
+ "FP": int(neg_hist[edge:].sum()),
512
+ "FN": int(pos_hist[:edge].sum()),
513
+ "TN": int(neg_hist[:edge].sum()),
514
+ }
515
+
516
+
517
+ def select_threshold_from_histograms(
518
+ pos_hist: np.ndarray,
519
+ neg_hist: np.ndarray,
520
+ criterion: str,
521
+ target_sensitivity: Optional[float] = None,
522
+ ) -> Tuple[int, float]:
523
+ """
524
+ Choose the threshold among the inner bin edges of the score histograms
525
+ and return (edge index, threshold).
526
+
527
+ youden: maximizes sensitivity + specificity - 1, the middle edge of the
528
+ best ones when several tie.
529
+ target_sensitivity: the highest edge whose sensitivity reaches the
530
+ target, so the best specificity under that constraint.
531
+ """
532
+ pos_hist, neg_hist = np.asarray(pos_hist), np.asarray(neg_hist)
533
+ num_bins = len(pos_hist)
534
+ positives, negatives = int(pos_hist.sum()), int(neg_hist.sum())
535
+ if positives == 0 or negatives == 0:
536
+ raise ValueError(
537
+ f"The validation data has a single class ({positives} positive, "
538
+ f"{negatives} negative): the threshold cannot be optimized"
539
+ )
540
+
541
+ # candidate thresholds, strictly between 0 and 1
542
+ edges = np.arange(1, num_bins)
543
+ # scores in bin k and above
544
+ positives_above = np.cumsum(pos_hist[::-1])[::-1]
545
+ negatives_above = np.cumsum(neg_hist[::-1])[::-1]
546
+ sensitivity = positives_above[edges] / positives
547
+ specificity = 1 - negatives_above[edges] / negatives
548
+
549
+ if criterion == "youden":
550
+ youden = sensitivity + specificity - 1
551
+ ties = np.flatnonzero(youden >= youden.max() - 1e-12)
552
+ edge = int(edges[ties[len(ties) // 2]])
553
+ elif criterion == "target_sensitivity":
554
+ target = float(target_sensitivity)
555
+ if not 0.0 < target <= 1.0:
556
+ raise ValueError(f"The target sensitivity must be in (0, 1] (got {target})")
557
+ # sensitivity decreases with the edge
558
+ reached = np.flatnonzero(sensitivity >= target - 1e-12)
559
+ if not len(reached):
560
+ raise ValueError(
561
+ f"No threshold reaches a validation sensitivity of {target}: "
562
+ f"{int(pos_hist[0])} of the {positives} positive case(s) score below "
563
+ f"the lowest candidate threshold {1 / num_bins:g}. Increase the number "
564
+ f"of bins or lower the target sensitivity"
565
+ )
566
+ edge = int(edges[reached.max()])
567
+ else:
568
+ raise ValueError(f"Unknown threshold criterion '{criterion}'")
569
+
570
+ return edge, float(_bin_edges(num_bins)[edge])
571
+
572
+
573
+ def optimize_federated_threshold(results, holdout_config: HoldoutConfig) -> Dict[str, Any]:
574
+ """
575
+ Choose the threshold on the pooled validation histograms of the final
576
+ round. Returns the selection with status "completed" (threshold, edge,
577
+ validation metrics) or "failed" (error), and the excluded clients.
578
+ """
579
+ completed, skipped, missing, failed = [], [], [], []
580
+
581
+ for client_proxy, eval_res in results:
582
+ metrics = eval_res.metrics or {}
583
+ status = metrics.get("threshold_validation_status")
584
+ name = str(metrics.get("holdout_client_id") or _client_name(client_proxy))
585
+
586
+ if status is None:
587
+ missing.append({
588
+ "client_id": name,
589
+ "error": "No validation data sent (the client MEDfl version may not support the optimized threshold)",
590
+ })
591
+ continue
592
+
593
+ summary = None
594
+ if metrics.get("threshold_validation"):
595
+ try:
596
+ summary = json.loads(metrics["threshold_validation"])
597
+ except Exception as exc:
598
+ failed.append({"client_id": name, "error": f"Invalid validation JSON: {exc}"})
599
+ continue
600
+
601
+ if status == HOLDOUT_COMPLETED and summary:
602
+ if int(summary.get("num_bins", -1)) != int(holdout_config.num_bins):
603
+ failed.append({
604
+ "client_id": summary.get("client_id", name),
605
+ "error": f"Used {summary.get('num_bins')} histogram bins instead of {holdout_config.num_bins}",
606
+ })
607
+ continue
608
+ completed.append(summary)
609
+ elif status == HOLDOUT_SKIPPED and summary:
610
+ skipped.append({
611
+ "client_id": summary.get("client_id", name),
612
+ "rows": summary["dataset"]["rows"],
613
+ "reason": summary.get("reason"),
614
+ })
615
+ elif status == HOLDOUT_MISSING:
616
+ missing.append({
617
+ "client_id": name,
618
+ "error": (summary or {}).get("reason") or metrics.get("threshold_validation_error"),
619
+ })
620
+ else:
621
+ failed.append({
622
+ "client_id": name,
623
+ "error": metrics.get("threshold_validation_error", "No validation result returned"),
624
+ })
625
+
626
+ criterion = holdout_config.threshold_criterion
627
+ selection: Dict[str, Any] = {
628
+ "criterion": criterion,
629
+ "criterion_label": THRESHOLD_CRITERIA.get(criterion, criterion),
630
+ "target_sensitivity": (
631
+ float(holdout_config.target_sensitivity) if criterion == "target_sensitivity" else None
632
+ ),
633
+ "method": f"pooled {int(holdout_config.num_bins)}-bin validation score histograms",
634
+ "num_bins": int(holdout_config.num_bins),
635
+ # the threshold is a bin edge
636
+ "resolution": 1.0 / int(holdout_config.num_bins),
637
+ "skipped_clients": skipped,
638
+ "missing_clients": missing,
639
+ "failed_clients": failed,
640
+ }
641
+
642
+ if not completed:
643
+ return {**selection, "status": "failed", "error": "No client returned validation data"}
644
+
645
+ pos_hist = np.sum([summary["pos_hist"] for summary in completed], axis=0)
646
+ neg_hist = np.sum([summary["neg_hist"] for summary in completed], axis=0)
647
+ positives, negatives = int(pos_hist.sum()), int(neg_hist.sum())
648
+ selection["validation"] = {
649
+ "clients": len(completed),
650
+ "rows": positives + negatives,
651
+ "positives": positives,
652
+ "negatives": negatives,
653
+ }
654
+
655
+ try:
656
+ edge, threshold = select_threshold_from_histograms(
657
+ pos_hist, neg_hist, criterion, holdout_config.target_sensitivity
658
+ )
659
+ except ValueError as exc:
660
+ return {**selection, "status": "failed", "error": str(exc)}
661
+
662
+ confusion = histogram_confusion(pos_hist, neg_hist, edge)
663
+ auc, _ = pooled_roc_auc(pos_hist, neg_hist)
664
+ return {
665
+ **selection,
666
+ "status": "completed",
667
+ "threshold": threshold,
668
+ "edge": edge,
669
+ "metrics": {"auc": auc, **confusion_metrics(confusion)},
670
+ "confusion_matrix": confusion,
671
+ }
672
+
673
+
331
674
  def aggregate_federated_holdout(
332
675
  results,
333
676
  failures,
@@ -390,9 +733,10 @@ def aggregate_federated_holdout(
390
733
  else:
391
734
  failed.append({"client_id": _client_name(failure[0]), "error": "Evaluation failed"})
392
735
 
736
+ threshold = float(holdout_config.threshold)
393
737
  result: Dict[str, Any] = {
394
738
  "backend": backend,
395
- "threshold": float(holdout_config.threshold),
739
+ "threshold": threshold,
396
740
  "num_bins": int(holdout_config.num_bins),
397
741
  "minimum_samples": int(holdout_config.minimum_samples),
398
742
  "participating_clients": len(completed),
@@ -401,6 +745,31 @@ def aggregate_federated_holdout(
401
745
  "failed_clients": failed,
402
746
  }
403
747
 
748
+ if holdout_config.threshold_source == "optimized":
749
+ # chosen on the validation histograms only, the holdout ones are not read here
750
+ selection = optimize_federated_threshold(results, holdout_config)
751
+ result["threshold_selection"] = selection
752
+ if selection["status"] != "completed":
753
+ # no fallback to the manual threshold
754
+ return {
755
+ **result,
756
+ "status": "failed",
757
+ "error": f"Threshold optimization failed: {selection['error']}",
758
+ }
759
+
760
+ threshold = selection["threshold"]
761
+ result["threshold"] = threshold
762
+ # the client metrics were computed at the manual threshold: recompute them exactly at the bin edge
763
+ for summary in completed:
764
+ summary["threshold"] = threshold
765
+ summary["confusion_matrix"] = histogram_confusion(
766
+ summary["pos_hist"], summary["neg_hist"], selection["edge"]
767
+ )
768
+ summary["metrics"] = {
769
+ "auc": summary["metrics"].get("auc"),
770
+ **confusion_metrics(summary["confusion_matrix"]),
771
+ }
772
+
404
773
  if not completed:
405
774
  return {
406
775
  **result,
@@ -442,7 +811,7 @@ def aggregate_federated_holdout(
442
811
  sum(summary["dataset"]["rows_without_target"] for summary in completed)
443
812
  ),
444
813
  },
445
- "threshold": float(holdout_config.threshold),
814
+ "threshold": threshold,
446
815
  "metrics": {"auc": auc, **confusion_metrics(confusion)},
447
816
  "confusion_matrix": confusion,
448
817
  "roc_curve": roc,
@@ -0,0 +1,12 @@
1
+ """
2
+ Name a real-world client reports to the server (properties, metrics, SHAP and holdout results).
3
+
4
+ The MEDfl agent ID (MEDFL_AGENT_ID, inherited from the agent that starts the client) when it is set,
5
+ so several agents on the same machine are told apart. Else the machine hostname, as before.
6
+ """
7
+ import os
8
+ import socket
9
+
10
+
11
+ def client_name() -> str:
12
+ return (os.getenv("MEDFL_AGENT_ID") or "").strip() or socket.gethostname()
@@ -3,7 +3,7 @@ import json
3
3
  import socket
4
4
  import platform
5
5
  import shutil
6
- from typing import Dict, Any, Optional
6
+ from typing import Dict, Any, Optional, Tuple
7
7
 
8
8
  import flwr as fl
9
9
  import numpy as np
@@ -31,6 +31,7 @@ try:
31
31
  except ImportError:
32
32
  GPUtil = None
33
33
 
34
+ from MEDfl.rw.identity import client_name
34
35
  from MEDfl.rw.xgboost.data import XGBoostDataModule
35
36
  from MEDfl.rw.xgboost.metrics import evaluate_predictions
36
37
  from MEDfl.rw.xgboost.serialization import (
@@ -374,7 +375,7 @@ class XGBoostFlowerClient(fl.client.Client):
374
375
  )
375
376
 
376
377
  return {
377
- "client_id": socket.gethostname(),
378
+ "client_id": client_name(),
378
379
  "feature_names": feature_names,
379
380
  "available_samples": available_samples,
380
381
  "sample_count": explained_samples,
@@ -457,7 +458,7 @@ class XGBoostFlowerClient(fl.client.Client):
457
458
  metrics = {
458
459
  "backend": "xgboost",
459
460
  "task": self.task,
460
- "hostname": socket.gethostname(),
461
+ "hostname": client_name(),
461
462
  "os_type": platform.system(),
462
463
  "num_train": int(self.metadata.get("num_train", 0)),
463
464
  "features": ",".join(self.metadata.get("features", [])),
@@ -574,7 +575,7 @@ class XGBoostFlowerClient(fl.client.Client):
574
575
  if bool(config.get("holdout_enabled", False)):
575
576
  if self.task != "binary":
576
577
  metrics["holdout_status"] = HOLDOUT_UNSUPPORTED
577
- metrics["holdout_client_id"] = socket.gethostname()
578
+ metrics["holdout_client_id"] = client_name()
578
579
  metrics["holdout_error"] = (
579
580
  "The holdout test supports binary classification "
580
581
  f"only (task is '{self.task}')"
@@ -587,8 +588,9 @@ class XGBoostFlowerClient(fl.client.Client):
587
588
  features=self.data_module.effective_features,
588
589
  target=self.data_module.effective_target,
589
590
  predict=self._predict_probabilities,
590
- client_id=socket.gethostname(),
591
+ client_id=client_name(),
591
592
  drop_missing_features=False,
593
+ validation=self._validation_predictions,
592
594
  )
593
595
  )
594
596
 
@@ -607,12 +609,21 @@ class XGBoostFlowerClient(fl.client.Client):
607
609
  )
608
610
  return np.asarray(self.booster.predict(matrix), dtype=float)
609
611
 
612
+ def _validation_predictions(self) -> Tuple[np.ndarray, np.ndarray]:
613
+ """(y_true, y_prob) of the validation split, for the optimized holdout threshold."""
614
+ if self.dval is None or int(self.metadata.get("num_val", 0)) == 0:
615
+ return np.empty((0,)), np.empty((0,))
616
+ return (
617
+ np.asarray(self.dval.get_label(), dtype=float),
618
+ np.asarray(self.booster.predict(self.dval), dtype=float),
619
+ )
620
+
610
621
  def get_properties(self, ins: GetPropertiesIns) -> GetPropertiesRes:
611
622
  """
612
623
  Return machine and dataset properties.
613
624
  """
614
625
 
615
- hostname = socket.gethostname()
626
+ hostname = client_name()
616
627
  os_type = platform.system()
617
628
 
618
629
  df = self.data_module.df
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: medfl
3
- Version: 2.0.10.dev8
3
+ Version: 2.0.10.dev10
4
4
  Summary: Python Open-source package for simulating federated learning and differential privacy
5
5
  Home-page: https://github.com/MEDomics-UdeS/MEDfl
6
6
  Author: MEDomics consortium
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: medfl
3
- Version: 2.0.10.dev8
3
+ Version: 2.0.10.dev10
4
4
  Summary: Python Open-source package for simulating federated learning and differential privacy
5
5
  Home-page: https://github.com/MEDomics-UdeS/MEDfl
6
6
  Author: MEDomics consortium
@@ -38,6 +38,7 @@ MEDfl/NetManager/node.py
38
38
  MEDfl/rw/__init__.py
39
39
  MEDfl/rw/client.py
40
40
  MEDfl/rw/holdout.py
41
+ MEDfl/rw/identity.py
41
42
  MEDfl/rw/model.py
42
43
  MEDfl/rw/server.py
43
44
  MEDfl/rw/strategy.py
@@ -14,7 +14,7 @@ with open('requirements.txt') as f:
14
14
 
15
15
  setup(
16
16
  name="medfl",
17
- version="2.0.10.dev8",
17
+ version="2.0.10.dev10",
18
18
  author="MEDomics consortium",
19
19
  author_email="medomics.info@gmail.com",
20
20
  description="Python Open-source package for simulating federated learning and differential privacy",
File without changes
File without changes
File without changes
File without changes