MEDfl 2.0.5.dev0__py3-none-any.whl → 2.0.5.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- MEDfl/rw/client.py +301 -367
- MEDfl/rw/model.py +3 -3
- MEDfl/rw/strategy.py +202 -324
- {MEDfl-2.0.5.dev0.dist-info → medfl-2.0.5.dev2.dist-info}/METADATA +14 -2
- {MEDfl-2.0.5.dev0.dist-info → medfl-2.0.5.dev2.dist-info}/RECORD +8 -8
- {MEDfl-2.0.5.dev0.dist-info → medfl-2.0.5.dev2.dist-info}/WHEEL +1 -1
- {MEDfl-2.0.5.dev0.dist-info → medfl-2.0.5.dev2.dist-info/licenses}/LICENSE +0 -0
- {MEDfl-2.0.5.dev0.dist-info → medfl-2.0.5.dev2.dist-info}/top_level.txt +0 -0
MEDfl/rw/client.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# client.py
|
|
1
|
+
# File: client.py
|
|
2
2
|
|
|
3
3
|
import argparse
|
|
4
4
|
import pandas as pd
|
|
@@ -21,11 +21,6 @@ try:
|
|
|
21
21
|
except ImportError:
|
|
22
22
|
GPUtil = None
|
|
23
23
|
|
|
24
|
-
try:
|
|
25
|
-
import xgboost as xgb
|
|
26
|
-
except Exception:
|
|
27
|
-
xgb = None
|
|
28
|
-
|
|
29
24
|
|
|
30
25
|
class DPConfig:
|
|
31
26
|
"""
|
|
@@ -51,356 +46,346 @@ class DPConfig:
|
|
|
51
46
|
self.secure_rng = secure_rng
|
|
52
47
|
|
|
53
48
|
|
|
54
|
-
def booster_to_parameters(bst):
|
|
55
|
-
# Send JSON so the server can merge trees safely
|
|
56
|
-
raw = bst.save_raw(raw_format="json")
|
|
57
|
-
# NumPyClient expects a list of numpy arrays
|
|
58
|
-
return [np.frombuffer(raw, dtype=np.uint8)]
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
def parameters_to_booster(parameters):
|
|
62
|
-
# Handle numpy uint8 array or raw bytes/bytearray
|
|
63
|
-
buf = parameters[0]
|
|
64
|
-
if isinstance(buf, (bytes, bytearray, memoryview)):
|
|
65
|
-
raw = bytes(buf)
|
|
66
|
-
else:
|
|
67
|
-
arr = np.asarray(buf, dtype=np.uint8)
|
|
68
|
-
raw = arr.tobytes()
|
|
69
|
-
|
|
70
|
-
booster = xgb.Booster()
|
|
71
|
-
# xgboost can load both JSON and binary buffers
|
|
72
|
-
booster.load_model(bytearray(raw))
|
|
73
|
-
return booster
|
|
74
|
-
|
|
75
|
-
|
|
76
49
|
class FlowerClient(fl.client.NumPyClient):
|
|
77
50
|
def __init__(
|
|
78
51
|
self,
|
|
79
52
|
server_address: str,
|
|
80
53
|
data_path: str = "data/data.csv",
|
|
81
54
|
dp_config: DPConfig = None,
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
55
|
+
# NEW (optional client overrides; do NOT remove old args)
|
|
56
|
+
val_frac: float | None = None,
|
|
57
|
+
test_frac: float | None = None,
|
|
58
|
+
id_col: str | None = None,
|
|
59
|
+
test_ids: str | None = None,
|
|
60
|
+
seed: int = 42,
|
|
85
61
|
):
|
|
86
62
|
self.server_address = server_address
|
|
87
|
-
self.
|
|
88
|
-
self.
|
|
89
|
-
self.
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
self.
|
|
93
|
-
|
|
94
|
-
# Load once;
|
|
95
|
-
self.
|
|
96
|
-
|
|
97
|
-
# Defaults
|
|
98
|
-
|
|
99
|
-
|
|
63
|
+
self.dp_config = dp_config
|
|
64
|
+
self.client_val_frac = val_frac
|
|
65
|
+
self.client_test_frac = test_frac
|
|
66
|
+
self.id_col = id_col
|
|
67
|
+
self.test_ids = test_ids
|
|
68
|
+
self.seed = seed
|
|
69
|
+
|
|
70
|
+
# Load the CSV once; actual column selection happens on first fit using server config
|
|
71
|
+
self._df = pd.read_csv(data_path)
|
|
72
|
+
|
|
73
|
+
# Defaults used only for get_properties BEFORE first fit (last column target)
|
|
74
|
+
self.feature_names = self._df.columns[:-1].tolist()
|
|
75
|
+
self.target_name = self._df.columns[-1]
|
|
76
|
+
self.label_counts = self._df[self.target_name].value_counts().to_dict()
|
|
77
|
+
self.classes = sorted(self.label_counts.keys())
|
|
78
|
+
|
|
79
|
+
# Tensors for metrics before first fit (fallback to all-but-last as features)
|
|
80
|
+
X_default = self._df.iloc[:, :-1].values
|
|
81
|
+
y_default = self._df.iloc[:, -1].values
|
|
82
|
+
self.X_tensor = torch.tensor(X_default, dtype=torch.float32)
|
|
83
|
+
self.y_tensor = torch.tensor(y_default, dtype=torch.float32)
|
|
84
|
+
|
|
85
|
+
# Placeholders; we lazily build loaders/model on the first fit when we see server config
|
|
86
|
+
self.train_loader: DataLoader | None = None
|
|
87
|
+
self.val_loader: DataLoader | None = None
|
|
88
|
+
self.test_loader: DataLoader | None = None
|
|
89
|
+
|
|
90
|
+
self.model: Net | None = None
|
|
91
|
+
self.criterion = nn.BCEWithLogitsLoss()
|
|
92
|
+
self.optimizer: optim.Optimizer | None = None
|
|
93
|
+
|
|
94
|
+
# Effective settings (filled at first fit)
|
|
95
|
+
self.effective_features: list[str] = self.feature_names[:]
|
|
96
|
+
self.effective_target: str = self.target_name
|
|
97
|
+
self.effective_val_frac: float = float(self.client_val_frac) if self.client_val_frac is not None else 0.0
|
|
98
|
+
self.effective_test_frac: float = float(self.client_test_frac) if self.client_test_frac is not None else 0.0
|
|
99
|
+
|
|
100
|
+
self._initialized = False
|
|
101
|
+
self._dp_attached = False # to avoid wrapping twice
|
|
102
|
+
|
|
103
|
+
# ---------- helpers
|
|
104
|
+
|
|
105
|
+
def _mk_loader(self, X: np.ndarray, y: np.ndarray, batch_size: int, shuffle: bool) -> DataLoader:
|
|
106
|
+
x_t = torch.tensor(X, dtype=torch.float32)
|
|
107
|
+
y_t = torch.tensor(y, dtype=torch.float32)
|
|
108
|
+
return DataLoader(TensorDataset(x_t, y_t), batch_size=batch_size, shuffle=shuffle)
|
|
109
|
+
|
|
110
|
+
def _lazy_init_from_server_config(self, config: dict):
|
|
111
|
+
"""
|
|
112
|
+
Build model and (train, val, test) loaders once, using:
|
|
113
|
+
- Server-enforced schema: config['features'] (comma-separated), config['target']
|
|
114
|
+
- Split fractions: client overrides win; else use server's val_fraction/test_fraction
|
|
115
|
+
- NEW: if config['test_ids'] is set (per-client from strategy), use ID-based split
|
|
116
|
+
"""
|
|
117
|
+
# ---------- schema from server (enforced if provided)
|
|
118
|
+
srv_features = (config.get("features") or "").strip()
|
|
119
|
+
srv_target = (config.get("target") or "").strip()
|
|
120
|
+
print(f"[Client] Initializing with server schema: features='{srv_features}', target='{srv_target}'")
|
|
121
|
+
|
|
122
|
+
if srv_target:
|
|
123
|
+
if srv_target not in self._df.columns:
|
|
124
|
+
raise ValueError(f"Server-specified target '{srv_target}' not in CSV columns {list(self._df.columns)}")
|
|
125
|
+
target_col = srv_target
|
|
126
|
+
else:
|
|
127
|
+
target_col = self._df.columns[-1] # fallback (keeps backward compatibility)
|
|
100
128
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
129
|
+
if srv_features:
|
|
130
|
+
feat_cols = [c.strip() for c in srv_features.split(",") if c.strip()]
|
|
131
|
+
missing = [c for c in feat_cols if c not in self._df.columns]
|
|
132
|
+
if missing:
|
|
133
|
+
raise ValueError(f"Server-specified feature(s) not found in CSV: {missing}")
|
|
134
|
+
else:
|
|
135
|
+
feat_cols = [c for c in self._df.columns if c != target_col]
|
|
136
|
+
|
|
137
|
+
# ---------- fractions: client overrides > server defaults > fallback (0.10/0.10)
|
|
138
|
+
srv_val = config.get("val_fraction", None)
|
|
139
|
+
srv_test = config.get("test_fraction", None)
|
|
140
|
+
val_frac = self.client_val_frac if self.client_val_frac is not None else (float(srv_val) if srv_val is not None else 0.10)
|
|
141
|
+
test_frac = self.client_test_frac if self.client_test_frac is not None else (float(srv_test) if srv_test is not None else 0.10)
|
|
142
|
+
|
|
143
|
+
if not (0.0 <= val_frac < 1.0):
|
|
144
|
+
raise ValueError(f"Invalid val_frac: {val_frac} (must be 0 <= val_frac < 1)")
|
|
145
|
+
|
|
146
|
+
# ---------- NEW: adopt test_ids / id_col from server config if provided
|
|
147
|
+
if (not self.test_ids or not self.test_ids.strip()) and config.get("test_ids"):
|
|
148
|
+
|
|
149
|
+
# strategy (per_client) can inject test_ids as list or CSV; normalize to CSV string
|
|
150
|
+
ti = config.get("test_ids")
|
|
151
|
+
if isinstance(ti, (list, tuple, set)):
|
|
152
|
+
self.test_ids = ",".join(str(x) for x in ti)
|
|
153
|
+
else:
|
|
154
|
+
self.test_ids = str(ti)
|
|
155
|
+
|
|
156
|
+
print(f"[Client] Using server-provided test_ids: {self.test_ids}")
|
|
157
|
+
|
|
158
|
+
if (not self.id_col) and config.get("id_col"):
|
|
159
|
+
self.id_col = str(config.get("id_col"))
|
|
160
|
+
|
|
161
|
+
# ---------- extract arrays with the enforced schema
|
|
162
|
+
X_all = self._df[feat_cols].values
|
|
163
|
+
y_all = self._df[target_col].values
|
|
164
|
+
|
|
165
|
+
# Keep tensors for global metrics logging (same behavior as before)
|
|
166
|
+
self.X_tensor = torch.tensor(X_all, dtype=torch.float32)
|
|
167
|
+
self.y_tensor = torch.tensor(y_all, dtype=torch.float32)
|
|
168
|
+
|
|
169
|
+
# ---------- split
|
|
170
|
+
if self.test_ids and self.test_ids.strip(): # ID-based mode (unchanged behavior, now also supports server-provided IDs)
|
|
171
|
+
print("[Client] Using ID-based test selection")
|
|
172
|
+
test_ids_list = [i.strip() for i in self.test_ids.split(',') if i.strip()]
|
|
173
|
+
|
|
174
|
+
if self.id_col and self.id_col in self._df.columns:
|
|
175
|
+
id_series = self._df[self.id_col]
|
|
176
|
+
# Align types between id_series and test_ids_list
|
|
177
|
+
if np.issubdtype(id_series.dtype, np.number):
|
|
178
|
+
test_ids_list = [int(i) for i in test_ids_list]
|
|
179
|
+
else:
|
|
180
|
+
test_ids_list = [str(i) for i in test_ids_list]
|
|
181
|
+
|
|
182
|
+
else:
|
|
183
|
+
print(f"[Client] Falling back to line numbers (index) as IDs since id_col='{self.id_col}' is invalid or not provided")
|
|
184
|
+
id_series = self._df.index
|
|
185
|
+
try:
|
|
186
|
+
test_ids_list = [int(i) for i in test_ids_list]
|
|
187
|
+
except ValueError:
|
|
188
|
+
raise ValueError("Test IDs must be integers when using line numbers as IDs")
|
|
189
|
+
|
|
190
|
+
test_mask = id_series.isin(test_ids_list)
|
|
191
|
+
if not test_mask.any():
|
|
192
|
+
print("[Client] Warning: No matching IDs found for test set; it will be empty")
|
|
193
|
+
|
|
194
|
+
X_test = X_all[test_mask]
|
|
195
|
+
y_test = y_all[test_mask]
|
|
196
|
+
X_trval = X_all[~test_mask]
|
|
197
|
+
y_trval = y_all[~test_mask]
|
|
198
|
+
|
|
199
|
+
actual_test_frac = len(y_test) / len(y_all) if len(y_all) > 0 else 0.0
|
|
200
|
+
if val_frac + actual_test_frac >= 1.0:
|
|
201
|
+
raise ValueError(f"Validation fraction {val_frac} + actual test fraction {actual_test_frac} >= 1.0")
|
|
202
|
+
|
|
203
|
+
self.effective_test_frac = actual_test_frac # For logging
|
|
204
|
+
|
|
205
|
+
else: # Fraction-based mode (existing)
|
|
206
|
+
if not (0.0 <= test_frac < 1.0 and (val_frac + test_frac) < 1.0):
|
|
207
|
+
raise ValueError(f"Invalid fractions: val={val_frac}, test={test_frac} (require 0 <= val,test < 1 and val+test < 1)")
|
|
208
|
+
|
|
209
|
+
strat_all = y_all if len(np.unique(y_all)) > 1 else None
|
|
210
|
+
X_trval, X_test, y_trval, y_test = train_test_split(
|
|
211
|
+
X_all, y_all, test_size=test_frac, random_state=self.seed, stratify=strat_all
|
|
212
|
+
)
|
|
107
213
|
|
|
108
|
-
#
|
|
109
|
-
|
|
110
|
-
|
|
214
|
+
# Split val from trval (common to both modes)
|
|
215
|
+
if val_frac > 0 and len(y_trval) > 0:
|
|
216
|
+
actual_test_frac = len(y_test) / len(y_all) if len(y_all) > 0 else 0.0
|
|
217
|
+
rel_val = val_frac / (1.0 - actual_test_frac) if (1.0 - actual_test_frac) > 0 else 0.0
|
|
218
|
+
strat_tr = y_trval if len(np.unique(y_trval)) > 1 else None
|
|
219
|
+
X_train, X_val, y_train, y_val = train_test_split(
|
|
220
|
+
X_trval, y_trval, test_size=rel_val, random_state=self.seed, stratify=strat_tr
|
|
221
|
+
)
|
|
222
|
+
else:
|
|
223
|
+
X_train, y_train = X_trval, y_trval
|
|
224
|
+
X_val, y_val = np.empty((0, X_all.shape[1])), np.empty((0,))
|
|
225
|
+
|
|
226
|
+
# ---------- build loaders
|
|
227
|
+
batch_size = self.dp_config.batch_size if self.dp_config else 32
|
|
228
|
+
self.train_loader = self._mk_loader(X_train, y_train, batch_size, shuffle=True)
|
|
229
|
+
self.val_loader = self._mk_loader(X_val, y_val, batch_size= batch_size, shuffle=False) if len(y_val) else None
|
|
230
|
+
self.test_loader = self._mk_loader(X_test, y_test, batch_size, shuffle=False)
|
|
231
|
+
|
|
232
|
+
# ---------- model/optimizer
|
|
233
|
+
input_dim = X_all.shape[1]
|
|
234
|
+
self.model = Net(input_dim)
|
|
235
|
+
self.optimizer = optim.SGD(self.model.parameters(), lr=0.01)
|
|
236
|
+
|
|
237
|
+
# ---------- attach DP (same behavior as before; only wraps the train loader)
|
|
238
|
+
if self.dp_config and not self._dp_attached:
|
|
111
239
|
try:
|
|
112
240
|
from opacus import PrivacyEngine
|
|
113
|
-
|
|
114
|
-
self.
|
|
115
|
-
(
|
|
116
|
-
self.model,
|
|
117
|
-
self.optimizer,
|
|
118
|
-
self.train_loader,
|
|
119
|
-
) = self.privacy_engine.make_private(
|
|
241
|
+
privacy_engine = PrivacyEngine()
|
|
242
|
+
(self.model, self.optimizer, self.train_loader) = privacy_engine.make_private(
|
|
120
243
|
module=self.model,
|
|
121
244
|
optimizer=self.optimizer,
|
|
122
245
|
data_loader=self.train_loader,
|
|
123
|
-
noise_multiplier=dp_config.noise_multiplier,
|
|
124
|
-
max_grad_norm=dp_config.max_grad_norm,
|
|
125
|
-
secure_rng=dp_config.secure_rng,
|
|
246
|
+
noise_multiplier=self.dp_config.noise_multiplier,
|
|
247
|
+
max_grad_norm=self.dp_config.max_grad_norm,
|
|
248
|
+
secure_rng=self.dp_config.secure_rng,
|
|
126
249
|
)
|
|
250
|
+
self._dp_attached = True
|
|
127
251
|
except ImportError:
|
|
128
252
|
print("Opacus non installé : exécution sans DP.")
|
|
129
253
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
if target_name not in df.columns:
|
|
138
|
-
raise ValueError(f"Target '{target_name}' not found in CSV columns: {list(df.columns)}")
|
|
139
|
-
|
|
140
|
-
# Clamp and sanitize test_size
|
|
141
|
-
try:
|
|
142
|
-
ts = float(test_size)
|
|
143
|
-
except Exception:
|
|
144
|
-
ts = 0.20
|
|
145
|
-
ts = max(1e-6, min(ts, 0.9))
|
|
146
|
-
|
|
147
|
-
# Build X/y from chosen target
|
|
148
|
-
X_df = df.drop(columns=[target_name])
|
|
149
|
-
y_series = df[target_name]
|
|
150
|
-
X_full = X_df.values
|
|
151
|
-
|
|
152
|
-
# If y isn't numeric, factorize to integers (keep class labels for metadata)
|
|
153
|
-
if not np.issubdtype(np.asarray(y_series).dtype, np.number):
|
|
154
|
-
y_vals, uniques = pd.factorize(y_series)
|
|
155
|
-
y_full = y_vals.astype(np.float32, copy=False)
|
|
156
|
-
classes = list(map(str, uniques))
|
|
157
|
-
label_counts = y_series.value_counts().to_dict()
|
|
254
|
+
# ---------- record effective settings
|
|
255
|
+
self.effective_features = feat_cols
|
|
256
|
+
self.effective_target = target_col
|
|
257
|
+
self.effective_val_frac = float(val_frac)
|
|
258
|
+
# effective_test_frac already set above if ID mode; otherwise use the input
|
|
259
|
+
if self.test_ids and self.test_ids.strip():
|
|
260
|
+
pass # Already set
|
|
158
261
|
else:
|
|
159
|
-
|
|
160
|
-
classes = sorted(pd.Series(y_full).unique().tolist())
|
|
161
|
-
label_counts = pd.Series(y_full).value_counts().to_dict()
|
|
162
|
-
|
|
163
|
-
# Heuristic for stratification (classification-like)
|
|
164
|
-
is_classif = np.unique(y_full).shape[0] <= 50
|
|
165
|
-
strat = y_full if is_classif else None
|
|
166
|
-
|
|
167
|
-
X_train, X_test, y_train, y_test = train_test_split(
|
|
168
|
-
X_full,
|
|
169
|
-
y_full,
|
|
170
|
-
test_size=ts,
|
|
171
|
-
random_state=42,
|
|
172
|
-
stratify=strat if strat is not None else None,
|
|
173
|
-
)
|
|
262
|
+
self.effective_test_frac = float(test_frac)
|
|
174
263
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
self.target_name = target_name
|
|
178
|
-
self.label_counts = label_counts
|
|
179
|
-
self.classes = classes
|
|
180
|
-
|
|
181
|
-
# --- Build per-model buffers ---
|
|
182
|
-
if self.model_type == "nn":
|
|
183
|
-
# Train tensors
|
|
184
|
-
self.X_tensor = torch.tensor(X_train, dtype=torch.float32)
|
|
185
|
-
self.y_tensor = torch.tensor(y_train, dtype=torch.float32)
|
|
186
|
-
|
|
187
|
-
# Test tensors
|
|
188
|
-
self.X_test_tensor = torch.tensor(X_test, dtype=torch.float32)
|
|
189
|
-
self.y_test_tensor = torch.tensor(y_test, dtype=torch.float32)
|
|
190
|
-
|
|
191
|
-
# DataLoaders
|
|
192
|
-
batch_size = getattr(dp_config, "batch_size", 32) if dp_config else 32
|
|
193
|
-
train_ds = TensorDataset(self.X_tensor, self.y_tensor)
|
|
194
|
-
self.train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True)
|
|
195
|
-
self.test_loader = DataLoader(
|
|
196
|
-
TensorDataset(self.X_test_tensor, self.y_test_tensor),
|
|
197
|
-
batch_size=batch_size,
|
|
198
|
-
shuffle=False,
|
|
199
|
-
)
|
|
264
|
+
self._initialized = True
|
|
265
|
+
print(f"[Client] Initialized with features={feat_cols}, target={target_col}, val={val_frac}, test={self.effective_test_frac}")
|
|
200
266
|
|
|
201
|
-
|
|
202
|
-
input_dim = X_train.shape[1]
|
|
203
|
-
if not hasattr(self, "model"):
|
|
204
|
-
self.model = Net(input_dim)
|
|
205
|
-
self.criterion = nn.BCEWithLogitsLoss()
|
|
206
|
-
self.optimizer = optim.SGD(self.model.parameters(), lr=0.01)
|
|
267
|
+
# ---------- FL API (unchanged behavior) ----------
|
|
207
268
|
|
|
208
|
-
else:
|
|
209
|
-
# XGBoost DMatrices
|
|
210
|
-
if xgb is None:
|
|
211
|
-
raise ImportError("xgboost is not installed. `pip install xgboost`")
|
|
212
|
-
|
|
213
|
-
self.X_np = X_train.astype(np.float32, copy=False) # train
|
|
214
|
-
self.y_np = y_train.astype(np.float32, copy=False)
|
|
215
|
-
self.dtrain = xgb.DMatrix(self.X_np, label=self.y_np)
|
|
216
|
-
|
|
217
|
-
self.X_np_test = X_test.astype(np.float32, copy=False) # test
|
|
218
|
-
self.y_np_test = y_test.astype(np.float32, copy=False)
|
|
219
|
-
self.dtest = xgb.DMatrix(self.X_np_test, label=self.y_np_test)
|
|
220
|
-
|
|
221
|
-
# Cold-start booster only if not present yet
|
|
222
|
-
if not hasattr(self, "bst"):
|
|
223
|
-
self.bst = xgb.train(self.xgb_params, self.dtrain, num_boost_round=1)
|
|
224
|
-
|
|
225
|
-
# Remember current prep so we can skip unnecessary rebuilds
|
|
226
|
-
self._prepared_key = (self.target_name, float(ts))
|
|
227
|
-
|
|
228
|
-
def _pick_target_and_frac(self, cfg):
|
|
229
|
-
"""Pick ('target', 'testFrac') from a small dict; support alternative keys."""
|
|
230
|
-
if not isinstance(cfg, dict):
|
|
231
|
-
return None, None
|
|
232
|
-
tgt = cfg.get("target") or cfg.get("Target") or cfg.get("label")
|
|
233
|
-
frac = cfg.get("testFrac", cfg.get("test_size", None))
|
|
234
|
-
try:
|
|
235
|
-
frac = float(frac) if frac is not None else None
|
|
236
|
-
except Exception:
|
|
237
|
-
frac = None
|
|
238
|
-
return tgt, frac
|
|
239
|
-
|
|
240
|
-
def _resolve_dataset_from_cfg(self, ds_cfg):
|
|
241
|
-
"""
|
|
242
|
-
Resolve (target, test_size) from datasetConfig:
|
|
243
|
-
- If isGlobal=True, take globalConfig.{target,testFrac}
|
|
244
|
-
- else take datasetConfig[hostname].{target,testFrac}
|
|
245
|
-
- fallbacks: keep current settings or defaults
|
|
246
|
-
"""
|
|
247
|
-
default_target = getattr(self, "target_name", self.df.columns[-1])
|
|
248
|
-
default_frac = getattr(self, "_prepared_key", (None, 0.2))[1] if hasattr(self, "_prepared_key") else 0.2
|
|
249
|
-
|
|
250
|
-
if not isinstance(ds_cfg, dict):
|
|
251
|
-
return default_target, default_frac
|
|
252
|
-
|
|
253
|
-
is_global = bool(ds_cfg.get("isGlobal"))
|
|
254
|
-
if is_global:
|
|
255
|
-
tgt, frac = self._pick_target_and_frac(ds_cfg.get("globalConfig", {}))
|
|
256
|
-
else:
|
|
257
|
-
host_cfg = ds_cfg.get(self.hostname)
|
|
258
|
-
if not isinstance(host_cfg, dict):
|
|
259
|
-
lower_map = {str(k).lower(): v for k, v in ds_cfg.items() if isinstance(v, dict)}
|
|
260
|
-
host_cfg = lower_map.get(self.hostname.lower())
|
|
261
|
-
tgt, frac = self._pick_target_and_frac(host_cfg or {})
|
|
262
|
-
|
|
263
|
-
target = tgt if tgt else default_target
|
|
264
|
-
test_size = frac if frac is not None else default_frac
|
|
265
|
-
test_size = max(1e-6, min(float(test_size), 0.9))
|
|
266
|
-
return target, test_size
|
|
267
|
-
|
|
268
|
-
def _ensure_prepared_from_config(self, config, dp_config=None):
|
|
269
|
-
"""Check config['datasetConfig']; rebuild splits if (target,test_size) changed."""
|
|
270
|
-
ds_cfg = config.get("dataset_config") if isinstance(config, dict) else None
|
|
271
|
-
if not isinstance(ds_cfg, dict):
|
|
272
|
-
return
|
|
273
|
-
|
|
274
|
-
target, ts = self._resolve_dataset_from_cfg(ds_cfg)
|
|
275
|
-
current = getattr(self, "_prepared_key", None)
|
|
276
|
-
desired = (target, float(ts))
|
|
277
|
-
if current != desired:
|
|
278
|
-
# Do not re-apply DP dynamically here
|
|
279
|
-
self._prepare_splits(target_name=target, test_size=ts, dp_config=None)
|
|
280
|
-
|
|
281
|
-
# --------------------------------------------------------------------------
|
|
282
|
-
# Federated API
|
|
283
|
-
# --------------------------------------------------------------------------
|
|
284
269
|
def get_parameters(self, config):
|
|
285
|
-
if self.
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
270
|
+
if not self._initialized:
|
|
271
|
+
try:
|
|
272
|
+
self._lazy_init_from_server_config(config if isinstance(config, dict) else {})
|
|
273
|
+
except Exception as e:
|
|
274
|
+
if not self._initialized:
|
|
275
|
+
self._lazy_init_from_server_config({})
|
|
276
|
+
return [val.cpu().numpy() for val in self.model.state_dict().values()]
|
|
289
277
|
|
|
290
278
|
def set_parameters(self, parameters):
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
self.model.load_state_dict(state_dict, strict=True)
|
|
295
|
-
else:
|
|
296
|
-
if parameters and len(parameters) > 0:
|
|
297
|
-
self.bst = parameters_to_booster(parameters)
|
|
279
|
+
params_dict = zip(self.model.state_dict().keys(), parameters)
|
|
280
|
+
state_dict = {k: torch.tensor(v) for k, v in params_dict}
|
|
281
|
+
self.model.load_state_dict(state_dict, strict=True)
|
|
298
282
|
|
|
299
283
|
def fit(self, parameters, config):
|
|
300
|
-
self.
|
|
284
|
+
if not self._initialized:
|
|
285
|
+
self._lazy_init_from_server_config(config)
|
|
301
286
|
|
|
302
|
-
|
|
303
|
-
self.
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
287
|
+
self.set_parameters(parameters)
|
|
288
|
+
self.model.train()
|
|
289
|
+
|
|
290
|
+
local_epochs = config.get("local_epochs", 5)
|
|
291
|
+
total_loss = 0.0
|
|
292
|
+
print(f"Training for {local_epochs} epochs...")
|
|
293
|
+
|
|
294
|
+
for epoch in range(local_epochs):
|
|
295
|
+
print(f"Epoch {epoch + 1}/{local_epochs}")
|
|
296
|
+
for X_batch, y_batch in self.train_loader:
|
|
297
|
+
self.optimizer.zero_grad()
|
|
298
|
+
outputs = self.model(X_batch)
|
|
299
|
+
loss = self.criterion(outputs.squeeze(), y_batch)
|
|
300
|
+
loss.backward()
|
|
301
|
+
self.optimizer.step()
|
|
302
|
+
total_loss += loss.item() * X_batch.size(0)
|
|
303
|
+
|
|
304
|
+
avg_loss = total_loss / (len(self.train_loader.dataset) * local_epochs)
|
|
305
|
+
|
|
306
|
+
with torch.no_grad():
|
|
307
|
+
logits = self.model(self.X_tensor).squeeze()
|
|
308
|
+
probs = torch.sigmoid(logits).cpu().numpy()
|
|
309
|
+
y_true = self.y_tensor.cpu().numpy()
|
|
310
|
+
binary_preds = (probs >= 0.5).astype(int)
|
|
311
|
+
try:
|
|
326
312
|
auc = roc_auc_score(y_true, probs)
|
|
313
|
+
except Exception:
|
|
314
|
+
auc = float("nan")
|
|
315
|
+
acc = accuracy_score(y_true, binary_preds)
|
|
327
316
|
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
self.
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
xgb_model=self.bst, # continue from global
|
|
342
|
-
)
|
|
343
|
-
preds = self.bst.predict(self.dtrain)
|
|
344
|
-
th = config.get("threshold", 0.5)
|
|
345
|
-
binary_preds = (preds >= th).astype(int)
|
|
346
|
-
acc = float((binary_preds == self.y_np).mean())
|
|
347
|
-
auc = float(roc_auc_score(self.y_np, preds)) if len(np.unique(self.y_np)) > 1 else 0.0
|
|
317
|
+
hostname = socket.gethostname()
|
|
318
|
+
os_type = platform.system()
|
|
319
|
+
metrics = {
|
|
320
|
+
"hostname": hostname,
|
|
321
|
+
"os_type": os_type,
|
|
322
|
+
"train_loss": avg_loss,
|
|
323
|
+
"train_accuracy": acc,
|
|
324
|
+
"train_auc": auc,
|
|
325
|
+
"features": ",".join(self.effective_features),
|
|
326
|
+
"target": self.effective_target,
|
|
327
|
+
"val_fraction": self.effective_val_frac,
|
|
328
|
+
"test_fraction": self.effective_test_frac,
|
|
329
|
+
}
|
|
348
330
|
|
|
349
|
-
|
|
350
|
-
return self.get_parameters(config), len(self.y_np), metrics
|
|
331
|
+
return self.get_parameters(config), len(self.train_loader.dataset), metrics
|
|
351
332
|
|
|
352
333
|
def evaluate(self, parameters, config):
|
|
353
|
-
self.
|
|
334
|
+
if not self._initialized:
|
|
335
|
+
self._lazy_init_from_server_config(config)
|
|
354
336
|
|
|
355
|
-
|
|
356
|
-
self.
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
th = config.get("threshold", 0.5)
|
|
373
|
-
binary_preds = [1 if p >= th else 0 for p in all_probs]
|
|
374
|
-
acc = accuracy_score(all_true, binary_preds)
|
|
337
|
+
self.set_parameters(parameters)
|
|
338
|
+
self.model.eval()
|
|
339
|
+
|
|
340
|
+
total_loss = 0.0
|
|
341
|
+
all_probs, all_true = [], []
|
|
342
|
+
with torch.no_grad():
|
|
343
|
+
for X_batch, y_batch in self.test_loader:
|
|
344
|
+
outputs = self.model(X_batch)
|
|
345
|
+
loss = self.criterion(outputs.squeeze(), y_batch)
|
|
346
|
+
total_loss += loss.item() * X_batch.size(0)
|
|
347
|
+
probs = torch.sigmoid(outputs.squeeze()).cpu().numpy()
|
|
348
|
+
all_probs.extend(probs.tolist())
|
|
349
|
+
all_true.extend(y_batch.cpu().numpy().tolist())
|
|
350
|
+
|
|
351
|
+
avg_loss = total_loss / len(self.test_loader.dataset) if len(self.test_loader.dataset) > 0 else 0.0
|
|
352
|
+
binary_preds = [1 if p >= 0.5 else 0 for p in all_probs]
|
|
353
|
+
try:
|
|
375
354
|
auc = roc_auc_score(all_true, all_probs)
|
|
376
|
-
|
|
377
|
-
|
|
355
|
+
except Exception:
|
|
356
|
+
auc = float("nan")
|
|
357
|
+
acc = accuracy_score(all_true, binary_preds)
|
|
378
358
|
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
# loss optional for XGB; return 0.0 to satisfy Flower
|
|
388
|
-
return 0.0, len(y_true), metrics
|
|
359
|
+
metrics = {
|
|
360
|
+
"eval_loss": avg_loss,
|
|
361
|
+
"eval_accuracy": acc,
|
|
362
|
+
"eval_auc": auc,
|
|
363
|
+
}
|
|
364
|
+
print(f"Evaluation metrics: {metrics}")
|
|
365
|
+
|
|
366
|
+
return float(avg_loss), len(self.test_loader.dataset), metrics
|
|
389
367
|
|
|
390
368
|
def get_properties(self, config):
|
|
391
369
|
hostname = socket.gethostname()
|
|
392
370
|
os_type = platform.system()
|
|
393
371
|
|
|
394
|
-
if self.
|
|
395
|
-
num_samples =
|
|
396
|
-
num_features = self.X_tensor.shape[1]
|
|
372
|
+
if self._initialized:
|
|
373
|
+
num_samples = int(self.X_tensor.shape[0])
|
|
374
|
+
num_features = int(self.X_tensor.shape[1])
|
|
375
|
+
features_str = ",".join(self.effective_features)
|
|
376
|
+
target_name = self.effective_target
|
|
377
|
+
label_counts = pd.Series(self.y_tensor.numpy()).value_counts().to_dict()
|
|
378
|
+
classes = sorted(label_counts.keys())
|
|
397
379
|
else:
|
|
398
|
-
num_samples = len(self.
|
|
399
|
-
num_features = self.
|
|
380
|
+
num_samples = len(self.X_tensor)
|
|
381
|
+
num_features = self.X_tensor.shape[1]
|
|
382
|
+
features_str = ",".join(self.feature_names)
|
|
383
|
+
target_name = self.target_name
|
|
384
|
+
label_counts = self.label_counts
|
|
385
|
+
classes = self.classes
|
|
400
386
|
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
dist_str = ",".join(f"{cls}:{cnt}" for cls, cnt in self.label_counts.items())
|
|
387
|
+
classes_str = ",".join(map(str, classes))
|
|
388
|
+
dist_str = ",".join(f"{cls}:{cnt}" for cls, cnt in label_counts.items())
|
|
404
389
|
|
|
405
390
|
cpu_physical = psutil.cpu_count(logical=False)
|
|
406
391
|
cpu_logical = psutil.cpu_count(logical=True)
|
|
@@ -419,7 +404,7 @@ class FlowerClient(fl.client.NumPyClient):
|
|
|
419
404
|
"num_samples": num_samples,
|
|
420
405
|
"num_features": num_features,
|
|
421
406
|
"features": features_str,
|
|
422
|
-
"target":
|
|
407
|
+
"target": target_name,
|
|
423
408
|
"classes": classes_str,
|
|
424
409
|
"label_distribution": dist_str,
|
|
425
410
|
"cpu_physical_cores": cpu_physical,
|
|
@@ -427,61 +412,10 @@ class FlowerClient(fl.client.NumPyClient):
|
|
|
427
412
|
"total_memory_gb": total_mem_gb,
|
|
428
413
|
"gpu_driver_present": str(driver_present),
|
|
429
414
|
"gpu_count": gpu_count,
|
|
430
|
-
"model_type": self.model_type,
|
|
431
415
|
}
|
|
432
416
|
|
|
433
417
|
def start(self) -> None:
|
|
434
|
-
fl.client.start_numpy_client(
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
if __name__ == "__main__":
|
|
438
|
-
parser = argparse.ArgumentParser(description="Flower client with NN/XGBoost + optional DP for NN")
|
|
439
|
-
parser.add_argument("--server_address", type=str, required=True, help="ex: 127.0.0.1:8080")
|
|
440
|
-
parser.add_argument("--data_path", type=str, default="data/data.csv", help="CSV path")
|
|
441
|
-
|
|
442
|
-
# Mode
|
|
443
|
-
parser.add_argument("--model", type=str, default="nn", choices=["nn", "xgb"], help="Client model type")
|
|
444
|
-
|
|
445
|
-
# DP (NN only)
|
|
446
|
-
parser.add_argument("--dp", action="store_true", help="Activer la confidentialité différentielle (NN uniquement)")
|
|
447
|
-
parser.add_argument("--noise_multiplier", type=float, default=1.0)
|
|
448
|
-
parser.add_argument("--max_grad_norm", type=float, default=1.0)
|
|
449
|
-
parser.add_argument("--batch_size", type=int, default=32)
|
|
450
|
-
|
|
451
|
-
# XGBoost params
|
|
452
|
-
parser.add_argument("--xgb_eta", type=float, default=0.1)
|
|
453
|
-
parser.add_argument("--xgb_max_depth", type=int, default=6)
|
|
454
|
-
parser.add_argument("--xgb_subsample", type=float, default=0.8)
|
|
455
|
-
parser.add_argument("--xgb_colsample_bytree", type=float, default=0.8)
|
|
456
|
-
parser.add_argument("--xgb_rounds", type=int, default=50)
|
|
457
|
-
args = parser.parse_args()
|
|
458
|
-
|
|
459
|
-
dp_config = None
|
|
460
|
-
if args.dp and args.model == "nn":
|
|
461
|
-
dp_config = DPConfig(
|
|
462
|
-
noise_multiplier=args.noise_multiplier,
|
|
463
|
-
max_grad_norm=args.max_grad_norm,
|
|
464
|
-
batch_size=args.batch_size,
|
|
418
|
+
fl.client.start_numpy_client(
|
|
419
|
+
server_address=self.server_address,
|
|
420
|
+
client=self,
|
|
465
421
|
)
|
|
466
|
-
|
|
467
|
-
xgb_params = None
|
|
468
|
-
if args.model == "xgb":
|
|
469
|
-
xgb_params = {
|
|
470
|
-
"objective": "binary:logistic",
|
|
471
|
-
"eval_metric": "logloss",
|
|
472
|
-
"eta": args.xgb_eta,
|
|
473
|
-
"max_depth": args.xgb_max_depth,
|
|
474
|
-
"subsample": args.xgb_subsample,
|
|
475
|
-
"colsample_bytree": args.xgb_colsample_bytree,
|
|
476
|
-
"tree_method": "hist",
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
client = FlowerClient(
|
|
480
|
-
server_address=args.server_address,
|
|
481
|
-
data_path=args.data_path,
|
|
482
|
-
dp_config=dp_config,
|
|
483
|
-
model_type=args.model,
|
|
484
|
-
xgb_params=xgb_params,
|
|
485
|
-
xgb_rounds=args.xgb_rounds if args.model == "xgb" else 0,
|
|
486
|
-
)
|
|
487
|
-
client.start()
|