flwr-nightly 1.19.0.dev20250528__py3-none-any.whl → 1.19.0.dev20250530__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flwr/cli/utils.py +11 -3
- flwr/client/mod/comms_mods.py +36 -17
- flwr/common/auth_plugin/auth_plugin.py +9 -3
- flwr/common/exit_handlers.py +30 -0
- flwr/common/inflatable_grpc_utils.py +27 -13
- flwr/common/message.py +11 -0
- flwr/common/record/array.py +10 -21
- flwr/common/record/arrayrecord.py +1 -1
- flwr/common/recorddict_compat.py +2 -2
- flwr/common/serde.py +1 -1
- flwr/proto/fleet_pb2.py +16 -16
- flwr/proto/fleet_pb2.pyi +5 -5
- flwr/proto/message_pb2.py +10 -10
- flwr/proto/message_pb2.pyi +4 -4
- flwr/proto/serverappio_pb2.py +26 -26
- flwr/proto/serverappio_pb2.pyi +5 -5
- flwr/server/app.py +45 -57
- flwr/server/superlink/fleet/grpc_rere/fleet_servicer.py +2 -0
- flwr/server/superlink/fleet/message_handler/message_handler.py +34 -7
- flwr/server/superlink/fleet/rest_rere/rest_api.py +5 -2
- flwr/server/superlink/linkstate/utils.py +8 -5
- flwr/server/superlink/serverappio/serverappio_servicer.py +45 -5
- flwr/server/superlink/utils.py +29 -0
- flwr/supercore/object_store/__init__.py +2 -1
- flwr/supercore/object_store/in_memory_object_store.py +9 -2
- flwr/supercore/object_store/object_store.py +12 -0
- flwr/superexec/exec_grpc.py +4 -3
- flwr/superexec/exec_user_auth_interceptor.py +33 -4
- flwr/supernode/start_client_internal.py +144 -170
- {flwr_nightly-1.19.0.dev20250528.dist-info → flwr_nightly-1.19.0.dev20250530.dist-info}/METADATA +1 -1
- {flwr_nightly-1.19.0.dev20250528.dist-info → flwr_nightly-1.19.0.dev20250530.dist-info}/RECORD +33 -33
- {flwr_nightly-1.19.0.dev20250528.dist-info → flwr_nightly-1.19.0.dev20250530.dist-info}/WHEEL +0 -0
- {flwr_nightly-1.19.0.dev20250528.dist-info → flwr_nightly-1.19.0.dev20250530.dist-info}/entry_points.txt +0 -0
@@ -19,7 +19,7 @@ from typing import Optional
|
|
19
19
|
|
20
20
|
from flwr.common.inflatable import get_object_id, is_valid_sha256_hash
|
21
21
|
|
22
|
-
from .object_store import ObjectStore
|
22
|
+
from .object_store import NoObjectInStoreError, ObjectStore
|
23
23
|
|
24
24
|
|
25
25
|
class InMemoryObjectStore(ObjectStore):
|
@@ -48,7 +48,9 @@ class InMemoryObjectStore(ObjectStore):
|
|
48
48
|
"""Put an object into the store."""
|
49
49
|
# Only allow adding the object if it has been preregistered
|
50
50
|
if object_id not in self.store:
|
51
|
-
raise
|
51
|
+
raise NoObjectInStoreError(
|
52
|
+
f"Object with ID '{object_id}' was not pre-registered."
|
53
|
+
)
|
52
54
|
|
53
55
|
# Verify object_id and object_content match
|
54
56
|
if self.verify:
|
@@ -71,6 +73,11 @@ class InMemoryObjectStore(ObjectStore):
|
|
71
73
|
|
72
74
|
def get_message_descendant_ids(self, msg_object_id: str) -> list[str]:
|
73
75
|
"""Retrieve the object IDs of all descendants of a given Message."""
|
76
|
+
if msg_object_id not in self.msg_children_objects_mapping:
|
77
|
+
raise NoObjectInStoreError(
|
78
|
+
f"No message registered in Object Store with ID '{msg_object_id}'. "
|
79
|
+
"Mapping to descendants could not be found."
|
80
|
+
)
|
74
81
|
return self.msg_children_objects_mapping[msg_object_id]
|
75
82
|
|
76
83
|
def get(self, object_id: str) -> Optional[bytes]:
|
@@ -19,6 +19,18 @@ import abc
|
|
19
19
|
from typing import Optional
|
20
20
|
|
21
21
|
|
22
|
+
class NoObjectInStoreError(Exception):
|
23
|
+
"""Error when trying to access an element in the ObjectStore that does not exist."""
|
24
|
+
|
25
|
+
def __init__(self, message: str):
|
26
|
+
super().__init__(message)
|
27
|
+
self.message = message
|
28
|
+
|
29
|
+
def __str__(self) -> str:
|
30
|
+
"""Return formatted exception message string."""
|
31
|
+
return f"NoObjectInStoreError: {self.message}"
|
32
|
+
|
33
|
+
|
22
34
|
class ObjectStore(abc.ABC):
|
23
35
|
"""Abstract base class for `ObjectStore` implementations.
|
24
36
|
|
flwr/superexec/exec_grpc.py
CHANGED
@@ -21,7 +21,7 @@ from typing import Optional
|
|
21
21
|
import grpc
|
22
22
|
|
23
23
|
from flwr.common import GRPC_MAX_MESSAGE_LENGTH
|
24
|
-
from flwr.common.auth_plugin import ExecAuthPlugin
|
24
|
+
from flwr.common.auth_plugin import ExecAuthPlugin, ExecAuthzPlugin
|
25
25
|
from flwr.common.event_log_plugin import EventLogWriterPlugin
|
26
26
|
from flwr.common.grpc import generic_create_grpc_server
|
27
27
|
from flwr.common.logger import log
|
@@ -45,6 +45,7 @@ def run_exec_api_grpc(
|
|
45
45
|
certificates: Optional[tuple[bytes, bytes, bytes]],
|
46
46
|
config: UserConfig,
|
47
47
|
auth_plugin: Optional[ExecAuthPlugin] = None,
|
48
|
+
authz_plugin: Optional[ExecAuthzPlugin] = None,
|
48
49
|
event_log_plugin: Optional[EventLogWriterPlugin] = None,
|
49
50
|
) -> grpc.Server:
|
50
51
|
"""Run Exec API (gRPC, request-response)."""
|
@@ -57,8 +58,8 @@ def run_exec_api_grpc(
|
|
57
58
|
auth_plugin=auth_plugin,
|
58
59
|
)
|
59
60
|
interceptors: list[grpc.ServerInterceptor] = []
|
60
|
-
if auth_plugin is not None:
|
61
|
-
interceptors.append(ExecUserAuthInterceptor(auth_plugin))
|
61
|
+
if auth_plugin is not None and authz_plugin is not None:
|
62
|
+
interceptors.append(ExecUserAuthInterceptor(auth_plugin, authz_plugin))
|
62
63
|
# Event log interceptor must be added after user auth interceptor
|
63
64
|
if event_log_plugin is not None:
|
64
65
|
interceptors.append(ExecEventLogInterceptor(event_log_plugin))
|
@@ -16,11 +16,11 @@
|
|
16
16
|
|
17
17
|
|
18
18
|
import contextvars
|
19
|
-
from typing import Any, Callable, Union
|
19
|
+
from typing import Any, Callable, Union
|
20
20
|
|
21
21
|
import grpc
|
22
22
|
|
23
|
-
from flwr.common.auth_plugin import ExecAuthPlugin
|
23
|
+
from flwr.common.auth_plugin import ExecAuthPlugin, ExecAuthzPlugin
|
24
24
|
from flwr.common.typing import UserInfo
|
25
25
|
from flwr.proto.exec_pb2 import ( # pylint: disable=E0611
|
26
26
|
GetAuthTokensRequest,
|
@@ -56,8 +56,10 @@ class ExecUserAuthInterceptor(grpc.ServerInterceptor): # type: ignore
|
|
56
56
|
def __init__(
|
57
57
|
self,
|
58
58
|
auth_plugin: ExecAuthPlugin,
|
59
|
+
authz_plugin: ExecAuthzPlugin,
|
59
60
|
):
|
60
61
|
self.auth_plugin = auth_plugin
|
62
|
+
self.authz_plugin = authz_plugin
|
61
63
|
|
62
64
|
def intercept_service(
|
63
65
|
self,
|
@@ -95,13 +97,40 @@ class ExecUserAuthInterceptor(grpc.ServerInterceptor): # type: ignore
|
|
95
97
|
metadata
|
96
98
|
)
|
97
99
|
if valid_tokens:
|
100
|
+
if user_info is None:
|
101
|
+
context.abort(
|
102
|
+
grpc.StatusCode.UNAUTHENTICATED,
|
103
|
+
"Tokens validated, but user info not found",
|
104
|
+
)
|
105
|
+
raise grpc.RpcError()
|
98
106
|
# Store user info in contextvars for authenticated users
|
99
|
-
shared_user_info.set(
|
107
|
+
shared_user_info.set(user_info)
|
108
|
+
# Check if the user is authorized
|
109
|
+
if not self.authz_plugin.verify_user_authorization(user_info):
|
110
|
+
context.abort(
|
111
|
+
grpc.StatusCode.PERMISSION_DENIED, "User not authorized"
|
112
|
+
)
|
113
|
+
raise grpc.RpcError()
|
100
114
|
return call(request, context) # type: ignore
|
101
115
|
|
102
116
|
# If the user is not authenticated, refresh tokens
|
103
|
-
tokens = self.auth_plugin.refresh_tokens(
|
117
|
+
tokens, user_info = self.auth_plugin.refresh_tokens(metadata)
|
104
118
|
if tokens is not None:
|
119
|
+
if user_info is None:
|
120
|
+
context.abort(
|
121
|
+
grpc.StatusCode.UNAUTHENTICATED,
|
122
|
+
"Tokens refreshed, but user info not found",
|
123
|
+
)
|
124
|
+
raise grpc.RpcError()
|
125
|
+
# Store user info in contextvars for authenticated users
|
126
|
+
shared_user_info.set(user_info)
|
127
|
+
# Check if the user is authorized
|
128
|
+
if not self.authz_plugin.verify_user_authorization(user_info):
|
129
|
+
context.abort(
|
130
|
+
grpc.StatusCode.PERMISSION_DENIED, "User not authorized"
|
131
|
+
)
|
132
|
+
raise grpc.RpcError()
|
133
|
+
|
105
134
|
context.send_initial_metadata(tokens)
|
106
135
|
return call(request, context) # type: ignore
|
107
136
|
|
@@ -40,7 +40,6 @@ from flwr.client.clientapp.clientappio_servicer import (
|
|
40
40
|
)
|
41
41
|
from flwr.client.grpc_adapter_client.connection import grpc_adapter
|
42
42
|
from flwr.client.grpc_rere_client.connection import grpc_request_response
|
43
|
-
from flwr.client.message_handler.message_handler import handle_control_message
|
44
43
|
from flwr.client.run_info_store import DeprecatedRunInfoStore
|
45
44
|
from flwr.common import GRPC_MAX_MESSAGE_LENGTH, Message
|
46
45
|
from flwr.common.address import parse_address
|
@@ -151,186 +150,161 @@ def start_client_internal(
|
|
151
150
|
|
152
151
|
runs: dict[int, Run] = {}
|
153
152
|
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
#
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
state.
|
174
|
-
|
175
|
-
|
176
|
-
node_config=node_config,
|
177
|
-
)
|
153
|
+
with _init_connection(
|
154
|
+
transport=transport,
|
155
|
+
server_address=server_address,
|
156
|
+
insecure=insecure,
|
157
|
+
root_certificates=root_certificates,
|
158
|
+
authentication_keys=authentication_keys,
|
159
|
+
max_retries=max_retries,
|
160
|
+
max_wait_time=max_wait_time,
|
161
|
+
) as conn:
|
162
|
+
receive, send, create_node, _, get_run, get_fab = conn
|
163
|
+
|
164
|
+
# Register node when connecting the first time
|
165
|
+
if run_info_store is None:
|
166
|
+
# Call create_node fn to register node
|
167
|
+
# and store node_id in state
|
168
|
+
if (node_id := create_node()) is None:
|
169
|
+
raise ValueError("Failed to register SuperNode with the SuperLink")
|
170
|
+
state.set_node_id(node_id)
|
171
|
+
run_info_store = DeprecatedRunInfoStore(
|
172
|
+
node_id=state.get_node_id(),
|
173
|
+
node_config=node_config,
|
174
|
+
)
|
178
175
|
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
log(
|
191
|
-
INFO,
|
192
|
-
"[RUN %s, ROUND %s]",
|
193
|
-
message.metadata.run_id,
|
194
|
-
message.metadata.group_id,
|
195
|
-
)
|
176
|
+
# pylint: disable=too-many-nested-blocks
|
177
|
+
while True:
|
178
|
+
try:
|
179
|
+
# Receive
|
180
|
+
message = receive()
|
181
|
+
if message is None:
|
182
|
+
time.sleep(3) # Wait for 3s before asking again
|
183
|
+
continue
|
184
|
+
|
185
|
+
log(INFO, "")
|
186
|
+
if len(message.metadata.group_id) > 0:
|
196
187
|
log(
|
197
188
|
INFO,
|
198
|
-
"
|
199
|
-
message.metadata.
|
200
|
-
message.metadata.
|
189
|
+
"[RUN %s, ROUND %s]",
|
190
|
+
message.metadata.run_id,
|
191
|
+
message.metadata.group_id,
|
201
192
|
)
|
193
|
+
log(
|
194
|
+
INFO,
|
195
|
+
"Received: %s message %s",
|
196
|
+
message.metadata.message_type,
|
197
|
+
message.metadata.message_id,
|
198
|
+
)
|
202
199
|
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
run_info_store.register_context(
|
226
|
-
run_id=run_id,
|
227
|
-
run=run,
|
228
|
-
flwr_path=flwr_path,
|
229
|
-
fab=fab,
|
230
|
-
)
|
200
|
+
# Get run info
|
201
|
+
run_id = message.metadata.run_id
|
202
|
+
if run_id not in runs:
|
203
|
+
runs[run_id] = get_run(run_id)
|
204
|
+
|
205
|
+
run: Run = runs[run_id]
|
206
|
+
if get_fab is not None and run.fab_hash:
|
207
|
+
fab = get_fab(run.fab_hash, run_id)
|
208
|
+
fab_id, fab_version = get_fab_metadata(fab.content)
|
209
|
+
else:
|
210
|
+
fab = None
|
211
|
+
fab_id, fab_version = run.fab_id, run.fab_version
|
212
|
+
|
213
|
+
run.fab_id, run.fab_version = fab_id, fab_version
|
214
|
+
|
215
|
+
# Register context for this run
|
216
|
+
run_info_store.register_context(
|
217
|
+
run_id=run_id,
|
218
|
+
run=run,
|
219
|
+
flwr_path=flwr_path,
|
220
|
+
fab=fab,
|
221
|
+
)
|
231
222
|
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
223
|
+
# Retrieve context for this run
|
224
|
+
context = run_info_store.retrieve_context(run_id=run_id)
|
225
|
+
# Create an error reply message that will never be used to prevent
|
226
|
+
# the used-before-assignment linting error
|
227
|
+
reply_message = Message(
|
228
|
+
Error(code=ErrorCode.UNKNOWN, reason="Unknown"),
|
229
|
+
reply_to=message,
|
230
|
+
)
|
240
231
|
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
# Generate SuperNode token
|
249
|
-
token = int.from_bytes(urandom(RUN_ID_NUM_BYTES), "little")
|
250
|
-
|
251
|
-
# Mode 1: SuperNode starts ClientApp as subprocess
|
252
|
-
start_subprocess = isolation == ISOLATION_MODE_SUBPROCESS
|
253
|
-
|
254
|
-
# Share Message and Context with servicer
|
255
|
-
clientappio_servicer.set_inputs(
|
256
|
-
clientapp_input=ClientAppInputs(
|
257
|
-
message=message,
|
258
|
-
context=context,
|
259
|
-
run=run,
|
260
|
-
fab=fab,
|
261
|
-
token=token,
|
262
|
-
),
|
263
|
-
token_returned=start_subprocess,
|
264
|
-
)
|
232
|
+
# Two isolation modes:
|
233
|
+
# 1. `subprocess`: SuperNode is starting the ClientApp
|
234
|
+
# process as a subprocess.
|
235
|
+
# 2. `process`: ClientApp process gets started separately
|
236
|
+
# (via `flwr-clientapp`), for example, in a separate
|
237
|
+
# Docker container.
|
265
238
|
|
266
|
-
|
267
|
-
|
268
|
-
io_address = (
|
269
|
-
f"{CLIENT_OCTET}:{_port}"
|
270
|
-
if _octet == SERVER_OCTET
|
271
|
-
else clientappio_api_address
|
272
|
-
)
|
273
|
-
# Start ClientApp subprocess
|
274
|
-
command = [
|
275
|
-
"flwr-clientapp",
|
276
|
-
"--clientappio-api-address",
|
277
|
-
io_address,
|
278
|
-
"--token",
|
279
|
-
str(token),
|
280
|
-
]
|
281
|
-
command.append("--insecure")
|
282
|
-
|
283
|
-
proc = mp_spawn_context.Process(
|
284
|
-
target=_run_flwr_clientapp,
|
285
|
-
args=(command, os.getpid()),
|
286
|
-
daemon=True,
|
287
|
-
)
|
288
|
-
proc.start()
|
289
|
-
proc.join()
|
290
|
-
else:
|
291
|
-
# Wait for output to become available
|
292
|
-
while not clientappio_servicer.has_outputs():
|
293
|
-
time.sleep(0.1)
|
294
|
-
|
295
|
-
outputs = clientappio_servicer.get_outputs()
|
296
|
-
reply_message, context = outputs.message, outputs.context
|
297
|
-
|
298
|
-
# Update node state
|
299
|
-
run_info_store.update_context(
|
300
|
-
run_id=run_id,
|
301
|
-
context=context,
|
302
|
-
)
|
239
|
+
# Generate SuperNode token
|
240
|
+
token = int.from_bytes(urandom(RUN_ID_NUM_BYTES), "little")
|
303
241
|
|
304
|
-
|
305
|
-
|
306
|
-
log(INFO, "Sent reply")
|
242
|
+
# Mode 1: SuperNode starts ClientApp as subprocess
|
243
|
+
start_subprocess = isolation == ISOLATION_MODE_SUBPROCESS
|
307
244
|
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
245
|
+
# Share Message and Context with servicer
|
246
|
+
clientappio_servicer.set_inputs(
|
247
|
+
clientapp_input=ClientAppInputs(
|
248
|
+
message=message,
|
249
|
+
context=context,
|
250
|
+
run=run,
|
251
|
+
fab=fab,
|
252
|
+
token=token,
|
253
|
+
),
|
254
|
+
token_returned=start_subprocess,
|
255
|
+
)
|
256
|
+
|
257
|
+
if start_subprocess:
|
258
|
+
_octet, _colon, _port = clientappio_api_address.rpartition(":")
|
259
|
+
io_address = (
|
260
|
+
f"{CLIENT_OCTET}:{_port}"
|
261
|
+
if _octet == SERVER_OCTET
|
262
|
+
else clientappio_api_address
|
315
263
|
)
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
264
|
+
# Start ClientApp subprocess
|
265
|
+
command = [
|
266
|
+
"flwr-clientapp",
|
267
|
+
"--clientappio-api-address",
|
268
|
+
io_address,
|
269
|
+
"--token",
|
270
|
+
str(token),
|
271
|
+
]
|
272
|
+
command.append("--insecure")
|
273
|
+
|
274
|
+
proc = mp_spawn_context.Process(
|
275
|
+
target=_run_flwr_clientapp,
|
276
|
+
args=(command, os.getpid()),
|
277
|
+
daemon=True,
|
278
|
+
)
|
279
|
+
proc.start()
|
280
|
+
proc.join()
|
281
|
+
else:
|
282
|
+
# Wait for output to become available
|
283
|
+
while not clientappio_servicer.has_outputs():
|
284
|
+
time.sleep(0.1)
|
285
|
+
|
286
|
+
outputs = clientappio_servicer.get_outputs()
|
287
|
+
reply_message, context = outputs.message, outputs.context
|
288
|
+
|
289
|
+
# Update node state
|
290
|
+
run_info_store.update_context(
|
291
|
+
run_id=run_id,
|
292
|
+
context=context,
|
293
|
+
)
|
294
|
+
|
295
|
+
# Send
|
296
|
+
send(reply_message)
|
297
|
+
log(INFO, "Sent reply")
|
298
|
+
|
299
|
+
except RunNotRunningException:
|
300
|
+
log(INFO, "")
|
301
|
+
log(
|
302
|
+
INFO,
|
303
|
+
"SuperNode aborted sending the reply message. "
|
304
|
+
"Run ID %s is not in `RUNNING` status.",
|
305
|
+
run_id,
|
306
|
+
)
|
307
|
+
log(INFO, "")
|
334
308
|
|
335
309
|
|
336
310
|
@contextmanager
|
{flwr_nightly-1.19.0.dev20250528.dist-info → flwr_nightly-1.19.0.dev20250530.dist-info}/METADATA
RENAMED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.3
|
2
2
|
Name: flwr-nightly
|
3
|
-
Version: 1.19.0.
|
3
|
+
Version: 1.19.0.dev20250530
|
4
4
|
Summary: Flower: A Friendly Federated AI Framework
|
5
5
|
License: Apache-2.0
|
6
6
|
Keywords: Artificial Intelligence,Federated AI,Federated Analytics,Federated Evaluation,Federated Learning,Flower,Machine Learning
|