flwr-nightly 1.15.0.dev20250104__py3-none-any.whl → 1.15.0.dev20250123__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flwr/cli/cli_user_auth_interceptor.py +6 -2
- flwr/cli/config_utils.py +23 -146
- flwr/cli/constant.py +27 -0
- flwr/cli/install.py +1 -1
- flwr/cli/log.py +17 -2
- flwr/cli/login/login.py +20 -5
- flwr/cli/ls.py +10 -2
- flwr/cli/run/run.py +20 -10
- flwr/cli/stop.py +9 -1
- flwr/cli/utils.py +4 -4
- flwr/client/app.py +36 -48
- flwr/client/clientapp/app.py +4 -6
- flwr/client/clientapp/utils.py +1 -1
- flwr/client/grpc_client/connection.py +0 -6
- flwr/client/grpc_rere_client/client_interceptor.py +19 -119
- flwr/client/grpc_rere_client/connection.py +34 -24
- flwr/client/grpc_rere_client/grpc_adapter.py +16 -0
- flwr/client/rest_client/connection.py +34 -26
- flwr/client/supernode/app.py +14 -20
- flwr/common/auth_plugin/auth_plugin.py +34 -23
- flwr/common/config.py +152 -15
- flwr/common/constant.py +11 -8
- flwr/common/exit/__init__.py +24 -0
- flwr/common/exit/exit.py +99 -0
- flwr/common/exit/exit_code.py +93 -0
- flwr/common/exit_handlers.py +24 -10
- flwr/common/grpc.py +161 -3
- flwr/common/logger.py +1 -1
- flwr/common/secure_aggregation/crypto/symmetric_encryption.py +45 -0
- flwr/common/serde.py +6 -4
- flwr/common/typing.py +20 -0
- flwr/proto/clientappio_pb2.py +13 -3
- flwr/proto/clientappio_pb2_grpc.py +63 -12
- flwr/proto/error_pb2.py +13 -3
- flwr/proto/error_pb2_grpc.py +20 -0
- flwr/proto/exec_pb2.py +27 -29
- flwr/proto/exec_pb2.pyi +27 -54
- flwr/proto/exec_pb2_grpc.py +105 -24
- flwr/proto/fab_pb2.py +13 -3
- flwr/proto/fab_pb2_grpc.py +20 -0
- flwr/proto/fleet_pb2.py +54 -31
- flwr/proto/fleet_pb2.pyi +84 -0
- flwr/proto/fleet_pb2_grpc.py +207 -28
- flwr/proto/fleet_pb2_grpc.pyi +26 -0
- flwr/proto/grpcadapter_pb2.py +14 -4
- flwr/proto/grpcadapter_pb2_grpc.py +35 -4
- flwr/proto/log_pb2.py +13 -3
- flwr/proto/log_pb2_grpc.py +20 -0
- flwr/proto/message_pb2.py +15 -5
- flwr/proto/message_pb2_grpc.py +20 -0
- flwr/proto/node_pb2.py +15 -5
- flwr/proto/node_pb2.pyi +1 -4
- flwr/proto/node_pb2_grpc.py +20 -0
- flwr/proto/recordset_pb2.py +18 -8
- flwr/proto/recordset_pb2_grpc.py +20 -0
- flwr/proto/run_pb2.py +16 -6
- flwr/proto/run_pb2_grpc.py +20 -0
- flwr/proto/serverappio_pb2.py +32 -14
- flwr/proto/serverappio_pb2.pyi +56 -0
- flwr/proto/serverappio_pb2_grpc.py +261 -44
- flwr/proto/serverappio_pb2_grpc.pyi +20 -0
- flwr/proto/simulationio_pb2.py +13 -3
- flwr/proto/simulationio_pb2_grpc.py +105 -24
- flwr/proto/task_pb2.py +13 -3
- flwr/proto/task_pb2_grpc.py +20 -0
- flwr/proto/transport_pb2.py +20 -10
- flwr/proto/transport_pb2_grpc.py +35 -4
- flwr/server/app.py +87 -38
- flwr/server/compat/app_utils.py +0 -1
- flwr/server/compat/driver_client_proxy.py +1 -2
- flwr/server/driver/grpc_driver.py +5 -2
- flwr/server/driver/inmemory_driver.py +2 -1
- flwr/server/serverapp/app.py +5 -6
- flwr/server/superlink/driver/serverappio_grpc.py +1 -1
- flwr/server/superlink/driver/serverappio_servicer.py +132 -14
- flwr/server/superlink/fleet/grpc_adapter/grpc_adapter_servicer.py +20 -88
- flwr/server/superlink/fleet/grpc_bidi/grpc_server.py +2 -165
- flwr/server/superlink/fleet/grpc_rere/fleet_servicer.py +38 -0
- flwr/server/superlink/fleet/grpc_rere/server_interceptor.py +95 -168
- flwr/server/superlink/fleet/message_handler/message_handler.py +66 -5
- flwr/server/superlink/fleet/rest_rere/rest_api.py +28 -3
- flwr/server/superlink/fleet/vce/vce_api.py +2 -2
- flwr/server/superlink/linkstate/in_memory_linkstate.py +40 -48
- flwr/server/superlink/linkstate/linkstate.py +15 -22
- flwr/server/superlink/linkstate/sqlite_linkstate.py +80 -99
- flwr/server/superlink/linkstate/utils.py +18 -8
- flwr/server/superlink/simulation/simulationio_grpc.py +1 -1
- flwr/server/utils/validator.py +9 -34
- flwr/simulation/app.py +4 -6
- flwr/simulation/legacy_app.py +4 -2
- flwr/simulation/run_simulation.py +1 -1
- flwr/superexec/exec_grpc.py +1 -1
- flwr/superexec/exec_servicer.py +23 -2
- {flwr_nightly-1.15.0.dev20250104.dist-info → flwr_nightly-1.15.0.dev20250123.dist-info}/METADATA +7 -7
- {flwr_nightly-1.15.0.dev20250104.dist-info → flwr_nightly-1.15.0.dev20250123.dist-info}/RECORD +98 -94
- {flwr_nightly-1.15.0.dev20250104.dist-info → flwr_nightly-1.15.0.dev20250123.dist-info}/LICENSE +0 -0
- {flwr_nightly-1.15.0.dev20250104.dist-info → flwr_nightly-1.15.0.dev20250123.dist-info}/WHEEL +0 -0
- {flwr_nightly-1.15.0.dev20250104.dist-info → flwr_nightly-1.15.0.dev20250123.dist-info}/entry_points.txt +0 -0
@@ -28,12 +28,11 @@ from ..driver.driver import Driver
|
|
28
28
|
class DriverClientProxy(ClientProxy):
|
29
29
|
"""Flower client proxy which delegates work using the Driver API."""
|
30
30
|
|
31
|
-
def __init__(self, node_id: int, driver: Driver,
|
31
|
+
def __init__(self, node_id: int, driver: Driver, run_id: int):
|
32
32
|
super().__init__(str(node_id))
|
33
33
|
self.node_id = node_id
|
34
34
|
self.driver = driver
|
35
35
|
self.run_id = run_id
|
36
|
-
self.anonymous = anonymous
|
37
36
|
|
38
37
|
def get_properties(
|
39
38
|
self,
|
@@ -24,7 +24,10 @@ from typing import Optional, cast
|
|
24
24
|
import grpc
|
25
25
|
|
26
26
|
from flwr.common import DEFAULT_TTL, Message, Metadata, RecordSet
|
27
|
-
from flwr.common.constant import
|
27
|
+
from flwr.common.constant import (
|
28
|
+
SERVERAPPIO_API_DEFAULT_CLIENT_ADDRESS,
|
29
|
+
SUPERLINK_NODE_ID,
|
30
|
+
)
|
28
31
|
from flwr.common.grpc import create_channel
|
29
32
|
from flwr.common.logger import log
|
30
33
|
from flwr.common.retry_invoker import _make_simple_grpc_retry_invoker, _wrap_stub
|
@@ -76,7 +79,7 @@ class GrpcDriver(Driver):
|
|
76
79
|
self._run: Optional[Run] = None
|
77
80
|
self._grpc_stub: Optional[ServerAppIoStub] = None
|
78
81
|
self._channel: Optional[grpc.Channel] = None
|
79
|
-
self.node = Node(node_id=
|
82
|
+
self.node = Node(node_id=SUPERLINK_NODE_ID)
|
80
83
|
self._retry_invoker = _make_simple_grpc_retry_invoker()
|
81
84
|
|
82
85
|
@property
|
@@ -22,6 +22,7 @@ from typing import Optional, cast
|
|
22
22
|
from uuid import UUID
|
23
23
|
|
24
24
|
from flwr.common import DEFAULT_TTL, Message, Metadata, RecordSet
|
25
|
+
from flwr.common.constant import SUPERLINK_NODE_ID
|
25
26
|
from flwr.common.serde import message_from_taskres, message_to_taskins
|
26
27
|
from flwr.common.typing import Run
|
27
28
|
from flwr.proto.node_pb2 import Node # pylint: disable=E0611
|
@@ -49,7 +50,7 @@ class InMemoryDriver(Driver):
|
|
49
50
|
self._run: Optional[Run] = None
|
50
51
|
self.state = state_factory.state()
|
51
52
|
self.pull_interval = pull_interval
|
52
|
-
self.node = Node(node_id=
|
53
|
+
self.node = Node(node_id=SUPERLINK_NODE_ID)
|
53
54
|
|
54
55
|
def _check_message(self, message: Message) -> None:
|
55
56
|
# Check if the message is valid
|
flwr/server/serverapp/app.py
CHANGED
@@ -16,7 +16,6 @@
|
|
16
16
|
|
17
17
|
|
18
18
|
import argparse
|
19
|
-
import sys
|
20
19
|
from logging import DEBUG, ERROR, INFO
|
21
20
|
from pathlib import Path
|
22
21
|
from queue import Queue
|
@@ -38,6 +37,7 @@ from flwr.common.constant import (
|
|
38
37
|
Status,
|
39
38
|
SubStatus,
|
40
39
|
)
|
40
|
+
from flwr.common.exit import ExitCode, flwr_exit
|
41
41
|
from flwr.common.logger import (
|
42
42
|
log,
|
43
43
|
mirror_output_to_queue,
|
@@ -75,12 +75,10 @@ def flwr_serverapp() -> None:
|
|
75
75
|
log(INFO, "Starting Flower ServerApp")
|
76
76
|
|
77
77
|
if not args.insecure:
|
78
|
-
|
79
|
-
|
80
|
-
"`flwr-serverapp` does not support TLS yet.
|
81
|
-
"Please use the '--insecure' flag.",
|
78
|
+
flwr_exit(
|
79
|
+
ExitCode.COMMON_TLS_NOT_SUPPORTED,
|
80
|
+
"`flwr-serverapp` does not support TLS yet.",
|
82
81
|
)
|
83
|
-
sys.exit(1)
|
84
82
|
|
85
83
|
log(
|
86
84
|
DEBUG,
|
@@ -117,6 +115,7 @@ def run_serverapp( # pylint: disable=R0914, disable=W0212, disable=R0915
|
|
117
115
|
log_uploader = None
|
118
116
|
success = True
|
119
117
|
hash_run_id = None
|
118
|
+
run_status = None
|
120
119
|
while True:
|
121
120
|
|
122
121
|
try:
|
@@ -21,6 +21,7 @@ from typing import Optional
|
|
21
21
|
import grpc
|
22
22
|
|
23
23
|
from flwr.common import GRPC_MAX_MESSAGE_LENGTH
|
24
|
+
from flwr.common.grpc import generic_create_grpc_server
|
24
25
|
from flwr.common.logger import log
|
25
26
|
from flwr.proto.serverappio_pb2_grpc import ( # pylint: disable=E0611
|
26
27
|
add_ServerAppIoServicer_to_server,
|
@@ -28,7 +29,6 @@ from flwr.proto.serverappio_pb2_grpc import ( # pylint: disable=E0611
|
|
28
29
|
from flwr.server.superlink.ffs.ffs_factory import FfsFactory
|
29
30
|
from flwr.server.superlink.linkstate import LinkStateFactory
|
30
31
|
|
31
|
-
from ..fleet.grpc_bidi.grpc_server import generic_create_grpc_server
|
32
32
|
from .serverappio_servicer import ServerAppIoServicer
|
33
33
|
|
34
34
|
|
@@ -16,14 +16,13 @@
|
|
16
16
|
|
17
17
|
|
18
18
|
import threading
|
19
|
-
import time
|
20
19
|
from logging import DEBUG, INFO
|
21
20
|
from typing import Optional
|
22
21
|
from uuid import UUID
|
23
22
|
|
24
23
|
import grpc
|
25
24
|
|
26
|
-
from flwr.common import ConfigsRecord
|
25
|
+
from flwr.common import ConfigsRecord, now
|
27
26
|
from flwr.common.constant import Status
|
28
27
|
from flwr.common.logger import log
|
29
28
|
from flwr.common.serde import (
|
@@ -31,6 +30,10 @@ from flwr.common.serde import (
|
|
31
30
|
context_to_proto,
|
32
31
|
fab_from_proto,
|
33
32
|
fab_to_proto,
|
33
|
+
message_from_proto,
|
34
|
+
message_from_taskres,
|
35
|
+
message_to_proto,
|
36
|
+
message_to_taskins,
|
34
37
|
run_status_from_proto,
|
35
38
|
run_status_to_proto,
|
36
39
|
run_to_proto,
|
@@ -57,10 +60,14 @@ from flwr.proto.run_pb2 import ( # pylint: disable=E0611
|
|
57
60
|
from flwr.proto.serverappio_pb2 import ( # pylint: disable=E0611
|
58
61
|
GetNodesRequest,
|
59
62
|
GetNodesResponse,
|
63
|
+
PullResMessagesRequest,
|
64
|
+
PullResMessagesResponse,
|
60
65
|
PullServerAppInputsRequest,
|
61
66
|
PullServerAppInputsResponse,
|
62
67
|
PullTaskResRequest,
|
63
68
|
PullTaskResResponse,
|
69
|
+
PushInsMessagesRequest,
|
70
|
+
PushInsMessagesResponse,
|
64
71
|
PushServerAppOutputsRequest,
|
65
72
|
PushServerAppOutputsResponse,
|
66
73
|
PushTaskInsRequest,
|
@@ -102,9 +109,7 @@ class ServerAppIoServicer(serverappio_pb2_grpc.ServerAppIoServicer):
|
|
102
109
|
)
|
103
110
|
|
104
111
|
all_ids: set[int] = state.get_nodes(request.run_id)
|
105
|
-
nodes: list[Node] = [
|
106
|
-
Node(node_id=node_id, anonymous=False) for node_id in all_ids
|
107
|
-
]
|
112
|
+
nodes: list[Node] = [Node(node_id=node_id) for node_id in all_ids]
|
108
113
|
return GetNodesResponse(nodes=nodes)
|
109
114
|
|
110
115
|
def CreateRun(
|
@@ -118,8 +123,9 @@ class ServerAppIoServicer(serverappio_pb2_grpc.ServerAppIoServicer):
|
|
118
123
|
ffs: Ffs = self.ffs_factory.ffs()
|
119
124
|
fab_hash = ffs.put(fab.content, {})
|
120
125
|
_raise_if(
|
121
|
-
fab_hash != fab.hash_str,
|
122
|
-
|
126
|
+
validation_error=fab_hash != fab.hash_str,
|
127
|
+
request_name="CreateRun",
|
128
|
+
detail=f"FAB ({fab.hash_str}) hash from request doesn't match contents",
|
123
129
|
)
|
124
130
|
else:
|
125
131
|
fab_hash = ""
|
@@ -150,17 +156,27 @@ class ServerAppIoServicer(serverappio_pb2_grpc.ServerAppIoServicer):
|
|
150
156
|
)
|
151
157
|
|
152
158
|
# Set pushed_at (timestamp in seconds)
|
153
|
-
pushed_at =
|
159
|
+
pushed_at = now().timestamp()
|
154
160
|
for task_ins in request.task_ins_list:
|
155
161
|
task_ins.task.pushed_at = pushed_at
|
156
162
|
|
157
163
|
# Validate request
|
158
|
-
_raise_if(
|
164
|
+
_raise_if(
|
165
|
+
validation_error=len(request.task_ins_list) == 0,
|
166
|
+
request_name="PushTaskIns",
|
167
|
+
detail="`task_ins_list` must not be empty",
|
168
|
+
)
|
159
169
|
for task_ins in request.task_ins_list:
|
160
170
|
validation_errors = validate_task_ins_or_res(task_ins)
|
161
|
-
_raise_if(bool(validation_errors), ", ".join(validation_errors))
|
162
171
|
_raise_if(
|
163
|
-
|
172
|
+
validation_error=bool(validation_errors),
|
173
|
+
request_name="PushTaskIns",
|
174
|
+
detail=", ".join(validation_errors),
|
175
|
+
)
|
176
|
+
_raise_if(
|
177
|
+
validation_error=request.run_id != task_ins.run_id,
|
178
|
+
request_name="PushTaskIns",
|
179
|
+
detail="`task_ins` has mismatched `run_id`",
|
164
180
|
)
|
165
181
|
|
166
182
|
# Store each TaskIns
|
@@ -173,6 +189,59 @@ class ServerAppIoServicer(serverappio_pb2_grpc.ServerAppIoServicer):
|
|
173
189
|
task_ids=[str(task_id) if task_id else "" for task_id in task_ids]
|
174
190
|
)
|
175
191
|
|
192
|
+
def PushMessages(
|
193
|
+
self, request: PushInsMessagesRequest, context: grpc.ServicerContext
|
194
|
+
) -> PushInsMessagesResponse:
|
195
|
+
"""Push a set of Messages."""
|
196
|
+
log(DEBUG, "ServerAppIoServicer.PushMessages")
|
197
|
+
|
198
|
+
# Init state
|
199
|
+
state: LinkState = self.state_factory.state()
|
200
|
+
|
201
|
+
# Abort if the run is not running
|
202
|
+
abort_if(
|
203
|
+
request.run_id,
|
204
|
+
[Status.PENDING, Status.STARTING, Status.FINISHED],
|
205
|
+
state,
|
206
|
+
context,
|
207
|
+
)
|
208
|
+
|
209
|
+
# Set pushed_at (timestamp in seconds)
|
210
|
+
pushed_at = now().timestamp()
|
211
|
+
|
212
|
+
# Validate request and insert in State
|
213
|
+
_raise_if(
|
214
|
+
validation_error=len(request.messages_list) == 0,
|
215
|
+
request_name="PushMessages",
|
216
|
+
detail="`messages_list` must not be empty",
|
217
|
+
)
|
218
|
+
message_ids: list[Optional[UUID]] = []
|
219
|
+
while request.messages_list:
|
220
|
+
message_proto = request.messages_list.pop(0)
|
221
|
+
message = message_from_proto(message_proto=message_proto)
|
222
|
+
task_ins = message_to_taskins(message=message)
|
223
|
+
task_ins.task.pushed_at = pushed_at
|
224
|
+
validation_errors = validate_task_ins_or_res(task_ins)
|
225
|
+
_raise_if(
|
226
|
+
validation_error=bool(validation_errors),
|
227
|
+
request_name="PushMessages",
|
228
|
+
detail=", ".join(validation_errors),
|
229
|
+
)
|
230
|
+
_raise_if(
|
231
|
+
validation_error=request.run_id != task_ins.run_id,
|
232
|
+
request_name="PushMessages",
|
233
|
+
detail="`task_ins` has mismatched `run_id`",
|
234
|
+
)
|
235
|
+
# Store
|
236
|
+
message_id: Optional[UUID] = state.store_task_ins(task_ins=task_ins)
|
237
|
+
message_ids.append(message_id)
|
238
|
+
|
239
|
+
return PushInsMessagesResponse(
|
240
|
+
message_ids=[
|
241
|
+
str(message_id) if message_id else "" for message_id in message_ids
|
242
|
+
]
|
243
|
+
)
|
244
|
+
|
176
245
|
def PullTaskRes(
|
177
246
|
self, request: PullTaskResRequest, context: grpc.ServicerContext
|
178
247
|
) -> PullTaskResResponse:
|
@@ -199,7 +268,9 @@ class ServerAppIoServicer(serverappio_pb2_grpc.ServerAppIoServicer):
|
|
199
268
|
# Validate request
|
200
269
|
for task_res in task_res_list:
|
201
270
|
_raise_if(
|
202
|
-
request.run_id != task_res.run_id,
|
271
|
+
validation_error=request.run_id != task_res.run_id,
|
272
|
+
request_name="PullTaskRes",
|
273
|
+
detail="`task_res` has mismatched `run_id`",
|
203
274
|
)
|
204
275
|
|
205
276
|
# Delete the TaskIns/TaskRes pairs if TaskRes is found
|
@@ -210,6 +281,52 @@ class ServerAppIoServicer(serverappio_pb2_grpc.ServerAppIoServicer):
|
|
210
281
|
|
211
282
|
return PullTaskResResponse(task_res_list=task_res_list)
|
212
283
|
|
284
|
+
def PullMessages(
|
285
|
+
self, request: PullResMessagesRequest, context: grpc.ServicerContext
|
286
|
+
) -> PullResMessagesResponse:
|
287
|
+
"""Pull a set of Messages."""
|
288
|
+
log(DEBUG, "ServerAppIoServicer.PullMessages")
|
289
|
+
|
290
|
+
# Init state
|
291
|
+
state: LinkState = self.state_factory.state()
|
292
|
+
|
293
|
+
# Abort if the run is not running
|
294
|
+
abort_if(
|
295
|
+
request.run_id,
|
296
|
+
[Status.PENDING, Status.STARTING, Status.FINISHED],
|
297
|
+
state,
|
298
|
+
context,
|
299
|
+
)
|
300
|
+
|
301
|
+
# Convert each task_id str to UUID
|
302
|
+
message_ids: set[UUID] = {
|
303
|
+
UUID(message_id) for message_id in request.message_ids
|
304
|
+
}
|
305
|
+
|
306
|
+
# Read from state
|
307
|
+
task_res_list: list[TaskRes] = state.get_task_res(task_ids=message_ids)
|
308
|
+
|
309
|
+
# Convert to Messages
|
310
|
+
messages_list = []
|
311
|
+
while task_res_list:
|
312
|
+
task_res = task_res_list.pop(0)
|
313
|
+
_raise_if(
|
314
|
+
validation_error=request.run_id != task_res.run_id,
|
315
|
+
request_name="PullMessages",
|
316
|
+
detail="`task_res` has mismatched `run_id`",
|
317
|
+
)
|
318
|
+
message = message_from_taskres(taskres=task_res)
|
319
|
+
messages_list.append(message_to_proto(message))
|
320
|
+
|
321
|
+
# Delete the TaskIns/TaskRes pairs if TaskRes is found
|
322
|
+
task_ins_ids_to_delete = {
|
323
|
+
UUID(task_res.task.ancestry[0]) for task_res in task_res_list
|
324
|
+
}
|
325
|
+
|
326
|
+
state.delete_tasks(task_ins_ids=task_ins_ids_to_delete)
|
327
|
+
|
328
|
+
return PullResMessagesResponse(messages_list=messages_list)
|
329
|
+
|
213
330
|
def GetRun(
|
214
331
|
self, request: GetRunRequest, context: grpc.ServicerContext
|
215
332
|
) -> GetRunResponse:
|
@@ -344,6 +461,7 @@ class ServerAppIoServicer(serverappio_pb2_grpc.ServerAppIoServicer):
|
|
344
461
|
return GetRunStatusResponse(run_status_dict=run_status_dict)
|
345
462
|
|
346
463
|
|
347
|
-
def _raise_if(validation_error: bool, detail: str) -> None:
|
464
|
+
def _raise_if(validation_error: bool, request_name: str, detail: str) -> None:
|
465
|
+
"""Raise a `ValueError` with a detailed message if a validation error occurs."""
|
348
466
|
if validation_error:
|
349
|
-
raise ValueError(f"Malformed
|
467
|
+
raise ValueError(f"Malformed {request_name}: {detail}")
|
@@ -15,7 +15,7 @@
|
|
15
15
|
"""Fleet API gRPC adapter servicer."""
|
16
16
|
|
17
17
|
|
18
|
-
from logging import DEBUG
|
18
|
+
from logging import DEBUG
|
19
19
|
from typing import Callable, TypeVar
|
20
20
|
|
21
21
|
import grpc
|
@@ -31,35 +31,30 @@ from flwr.common.constant import (
|
|
31
31
|
from flwr.common.logger import log
|
32
32
|
from flwr.common.version import package_name, package_version
|
33
33
|
from flwr.proto import grpcadapter_pb2_grpc # pylint: disable=E0611
|
34
|
-
from flwr.proto.fab_pb2 import GetFabRequest
|
34
|
+
from flwr.proto.fab_pb2 import GetFabRequest # pylint: disable=E0611
|
35
35
|
from flwr.proto.fleet_pb2 import ( # pylint: disable=E0611
|
36
36
|
CreateNodeRequest,
|
37
|
-
CreateNodeResponse,
|
38
37
|
DeleteNodeRequest,
|
39
|
-
DeleteNodeResponse,
|
40
38
|
PingRequest,
|
41
|
-
|
42
|
-
|
43
|
-
PullTaskInsResponse,
|
44
|
-
PushTaskResRequest,
|
45
|
-
PushTaskResResponse,
|
39
|
+
PullMessagesRequest,
|
40
|
+
PushMessagesRequest,
|
46
41
|
)
|
47
42
|
from flwr.proto.grpcadapter_pb2 import MessageContainer # pylint: disable=E0611
|
48
|
-
from flwr.proto.run_pb2 import GetRunRequest
|
49
|
-
|
50
|
-
from
|
51
|
-
from flwr.server.superlink.linkstate import LinkStateFactory
|
43
|
+
from flwr.proto.run_pb2 import GetRunRequest # pylint: disable=E0611
|
44
|
+
|
45
|
+
from ..grpc_rere.fleet_servicer import FleetServicer
|
52
46
|
|
53
47
|
T = TypeVar("T", bound=GrpcMessage)
|
54
48
|
|
55
49
|
|
56
50
|
def _handle(
|
57
51
|
msg_container: MessageContainer,
|
52
|
+
context: grpc.ServicerContext,
|
58
53
|
request_type: type[T],
|
59
|
-
handler: Callable[[T], GrpcMessage],
|
54
|
+
handler: Callable[[T, grpc.ServicerContext], GrpcMessage],
|
60
55
|
) -> MessageContainer:
|
61
56
|
req = request_type.FromString(msg_container.grpc_message_content)
|
62
|
-
res = handler(req)
|
57
|
+
res = handler(req, context)
|
63
58
|
res_cls = res.__class__
|
64
59
|
return MessageContainer(
|
65
60
|
metadata={
|
@@ -74,89 +69,26 @@ def _handle(
|
|
74
69
|
)
|
75
70
|
|
76
71
|
|
77
|
-
class GrpcAdapterServicer(grpcadapter_pb2_grpc.GrpcAdapterServicer):
|
72
|
+
class GrpcAdapterServicer(grpcadapter_pb2_grpc.GrpcAdapterServicer, FleetServicer):
|
78
73
|
"""Fleet API via GrpcAdapter servicer."""
|
79
74
|
|
80
|
-
def __init__(
|
81
|
-
self, state_factory: LinkStateFactory, ffs_factory: FfsFactory
|
82
|
-
) -> None:
|
83
|
-
self.state_factory = state_factory
|
84
|
-
self.ffs_factory = ffs_factory
|
85
|
-
|
86
75
|
def SendReceive( # pylint: disable=too-many-return-statements
|
87
76
|
self, request: MessageContainer, context: grpc.ServicerContext
|
88
77
|
) -> MessageContainer:
|
89
78
|
"""."""
|
90
79
|
log(DEBUG, "GrpcAdapterServicer.SendReceive")
|
91
80
|
if request.grpc_message_name == CreateNodeRequest.__qualname__:
|
92
|
-
return _handle(request, CreateNodeRequest, self.
|
81
|
+
return _handle(request, context, CreateNodeRequest, self.CreateNode)
|
93
82
|
if request.grpc_message_name == DeleteNodeRequest.__qualname__:
|
94
|
-
return _handle(request, DeleteNodeRequest, self.
|
83
|
+
return _handle(request, context, DeleteNodeRequest, self.DeleteNode)
|
95
84
|
if request.grpc_message_name == PingRequest.__qualname__:
|
96
|
-
return _handle(request, PingRequest, self.
|
97
|
-
if request.grpc_message_name == PullTaskInsRequest.__qualname__:
|
98
|
-
return _handle(request, PullTaskInsRequest, self._pull_task_ins)
|
99
|
-
if request.grpc_message_name == PushTaskResRequest.__qualname__:
|
100
|
-
return _handle(request, PushTaskResRequest, self._push_task_res)
|
85
|
+
return _handle(request, context, PingRequest, self.Ping)
|
101
86
|
if request.grpc_message_name == GetRunRequest.__qualname__:
|
102
|
-
return _handle(request, GetRunRequest, self.
|
87
|
+
return _handle(request, context, GetRunRequest, self.GetRun)
|
103
88
|
if request.grpc_message_name == GetFabRequest.__qualname__:
|
104
|
-
return _handle(request, GetFabRequest, self.
|
89
|
+
return _handle(request, context, GetFabRequest, self.GetFab)
|
90
|
+
if request.grpc_message_name == PullMessagesRequest.__qualname__:
|
91
|
+
return _handle(request, context, PullMessagesRequest, self.PullMessages)
|
92
|
+
if request.grpc_message_name == PushMessagesRequest.__qualname__:
|
93
|
+
return _handle(request, context, PushMessagesRequest, self.PushMessages)
|
105
94
|
raise ValueError(f"Invalid grpc_message_name: {request.grpc_message_name}")
|
106
|
-
|
107
|
-
def _create_node(self, request: CreateNodeRequest) -> CreateNodeResponse:
|
108
|
-
"""."""
|
109
|
-
log(INFO, "GrpcAdapter.CreateNode")
|
110
|
-
return message_handler.create_node(
|
111
|
-
request=request,
|
112
|
-
state=self.state_factory.state(),
|
113
|
-
)
|
114
|
-
|
115
|
-
def _delete_node(self, request: DeleteNodeRequest) -> DeleteNodeResponse:
|
116
|
-
"""."""
|
117
|
-
log(INFO, "GrpcAdapter.DeleteNode")
|
118
|
-
return message_handler.delete_node(
|
119
|
-
request=request,
|
120
|
-
state=self.state_factory.state(),
|
121
|
-
)
|
122
|
-
|
123
|
-
def _ping(self, request: PingRequest) -> PingResponse:
|
124
|
-
"""."""
|
125
|
-
log(DEBUG, "GrpcAdapter.Ping")
|
126
|
-
return message_handler.ping(
|
127
|
-
request=request,
|
128
|
-
state=self.state_factory.state(),
|
129
|
-
)
|
130
|
-
|
131
|
-
def _pull_task_ins(self, request: PullTaskInsRequest) -> PullTaskInsResponse:
|
132
|
-
"""Pull TaskIns."""
|
133
|
-
log(INFO, "GrpcAdapter.PullTaskIns")
|
134
|
-
return message_handler.pull_task_ins(
|
135
|
-
request=request,
|
136
|
-
state=self.state_factory.state(),
|
137
|
-
)
|
138
|
-
|
139
|
-
def _push_task_res(self, request: PushTaskResRequest) -> PushTaskResResponse:
|
140
|
-
"""Push TaskRes."""
|
141
|
-
log(INFO, "GrpcAdapter.PushTaskRes")
|
142
|
-
return message_handler.push_task_res(
|
143
|
-
request=request,
|
144
|
-
state=self.state_factory.state(),
|
145
|
-
)
|
146
|
-
|
147
|
-
def _get_run(self, request: GetRunRequest) -> GetRunResponse:
|
148
|
-
"""Get run information."""
|
149
|
-
log(INFO, "GrpcAdapter.GetRun")
|
150
|
-
return message_handler.get_run(
|
151
|
-
request=request,
|
152
|
-
state=self.state_factory.state(),
|
153
|
-
)
|
154
|
-
|
155
|
-
def _get_fab(self, request: GetFabRequest) -> GetFabResponse:
|
156
|
-
"""Get FAB."""
|
157
|
-
log(INFO, "GrpcAdapter.GetFab")
|
158
|
-
return message_handler.get_fab(
|
159
|
-
request=request,
|
160
|
-
ffs=self.ffs_factory.ffs(),
|
161
|
-
state=self.state_factory.state(),
|
162
|
-
)
|
@@ -15,49 +15,19 @@
|
|
15
15
|
"""Implements utility function to create a gRPC server."""
|
16
16
|
|
17
17
|
|
18
|
-
import
|
19
|
-
import sys
|
20
|
-
from collections.abc import Sequence
|
21
|
-
from logging import ERROR
|
22
|
-
from typing import Any, Callable, Optional, Union
|
18
|
+
from typing import Optional
|
23
19
|
|
24
20
|
import grpc
|
25
21
|
|
26
22
|
from flwr.common import GRPC_MAX_MESSAGE_LENGTH
|
27
|
-
from flwr.common.
|
28
|
-
from flwr.common.logger import log
|
23
|
+
from flwr.common.grpc import generic_create_grpc_server
|
29
24
|
from flwr.proto.transport_pb2_grpc import ( # pylint: disable=E0611
|
30
25
|
add_FlowerServiceServicer_to_server,
|
31
26
|
)
|
32
27
|
from flwr.server.client_manager import ClientManager
|
33
|
-
from flwr.server.superlink.driver.serverappio_servicer import ServerAppIoServicer
|
34
|
-
from flwr.server.superlink.fleet.grpc_adapter.grpc_adapter_servicer import (
|
35
|
-
GrpcAdapterServicer,
|
36
|
-
)
|
37
28
|
from flwr.server.superlink.fleet.grpc_bidi.flower_service_servicer import (
|
38
29
|
FlowerServiceServicer,
|
39
30
|
)
|
40
|
-
from flwr.server.superlink.fleet.grpc_rere.fleet_servicer import FleetServicer
|
41
|
-
|
42
|
-
INVALID_CERTIFICATES_ERR_MSG = """
|
43
|
-
When setting any of root_certificate, certificate, or private_key,
|
44
|
-
all of them need to be set.
|
45
|
-
"""
|
46
|
-
|
47
|
-
AddServicerToServerFn = Callable[..., Any]
|
48
|
-
|
49
|
-
|
50
|
-
def valid_certificates(certificates: tuple[bytes, bytes, bytes]) -> bool:
|
51
|
-
"""Validate certificates tuple."""
|
52
|
-
is_valid = (
|
53
|
-
all(isinstance(certificate, bytes) for certificate in certificates)
|
54
|
-
and len(certificates) == 3
|
55
|
-
)
|
56
|
-
|
57
|
-
if not is_valid:
|
58
|
-
log(ERROR, INVALID_CERTIFICATES_ERR_MSG)
|
59
|
-
|
60
|
-
return is_valid
|
61
31
|
|
62
32
|
|
63
33
|
def start_grpc_server( # pylint: disable=too-many-arguments,R0917
|
@@ -154,136 +124,3 @@ def start_grpc_server( # pylint: disable=too-many-arguments,R0917
|
|
154
124
|
server.start()
|
155
125
|
|
156
126
|
return server
|
157
|
-
|
158
|
-
|
159
|
-
def generic_create_grpc_server( # pylint: disable=too-many-arguments,R0917
|
160
|
-
servicer_and_add_fn: Union[
|
161
|
-
tuple[FleetServicer, AddServicerToServerFn],
|
162
|
-
tuple[GrpcAdapterServicer, AddServicerToServerFn],
|
163
|
-
tuple[FlowerServiceServicer, AddServicerToServerFn],
|
164
|
-
tuple[ServerAppIoServicer, AddServicerToServerFn],
|
165
|
-
],
|
166
|
-
server_address: str,
|
167
|
-
max_concurrent_workers: int = 1000,
|
168
|
-
max_message_length: int = GRPC_MAX_MESSAGE_LENGTH,
|
169
|
-
keepalive_time_ms: int = 210000,
|
170
|
-
certificates: Optional[tuple[bytes, bytes, bytes]] = None,
|
171
|
-
interceptors: Optional[Sequence[grpc.ServerInterceptor]] = None,
|
172
|
-
) -> grpc.Server:
|
173
|
-
"""Create a gRPC server with a single servicer.
|
174
|
-
|
175
|
-
Parameters
|
176
|
-
----------
|
177
|
-
servicer_and_add_fn : tuple
|
178
|
-
A tuple holding a servicer implementation and a matching
|
179
|
-
add_Servicer_to_server function.
|
180
|
-
server_address : str
|
181
|
-
Server address in the form of HOST:PORT e.g. "[::]:8080"
|
182
|
-
max_concurrent_workers : int
|
183
|
-
Maximum number of clients the server can process before returning
|
184
|
-
RESOURCE_EXHAUSTED status (default: 1000)
|
185
|
-
max_message_length : int
|
186
|
-
Maximum message length that the server can send or receive.
|
187
|
-
Int valued in bytes. -1 means unlimited. (default: GRPC_MAX_MESSAGE_LENGTH)
|
188
|
-
keepalive_time_ms : int
|
189
|
-
Flower uses a default gRPC keepalive time of 210000ms (3 minutes 30 seconds)
|
190
|
-
because some cloud providers (for example, Azure) agressively clean up idle
|
191
|
-
TCP connections by terminating them after some time (4 minutes in the case
|
192
|
-
of Azure). Flower does not use application-level keepalive signals and relies
|
193
|
-
on the assumption that the transport layer will fail in cases where the
|
194
|
-
connection is no longer active. `keepalive_time_ms` can be used to customize
|
195
|
-
the keepalive interval for specific environments. The default Flower gRPC
|
196
|
-
keepalive of 210000 ms (3 minutes 30 seconds) ensures that Flower can keep
|
197
|
-
the long running streaming connection alive in most environments. The actual
|
198
|
-
gRPC default of this setting is 7200000 (2 hours), which results in dropped
|
199
|
-
connections in some cloud environments.
|
200
|
-
|
201
|
-
These settings are related to the issue described here:
|
202
|
-
- https://github.com/grpc/proposal/blob/master/A8-client-side-keepalive.md
|
203
|
-
- https://github.com/grpc/grpc/blob/master/doc/keepalive.md
|
204
|
-
- https://grpc.io/docs/guides/performance/
|
205
|
-
|
206
|
-
Mobile Flower clients may choose to increase this value if their server
|
207
|
-
environment allows long-running idle TCP connections.
|
208
|
-
(default: 210000)
|
209
|
-
certificates : Tuple[bytes, bytes, bytes] (default: None)
|
210
|
-
Tuple containing root certificate, server certificate, and private key to
|
211
|
-
start a secure SSL-enabled server. The tuple is expected to have three bytes
|
212
|
-
elements in the following order:
|
213
|
-
|
214
|
-
* CA certificate.
|
215
|
-
* server certificate.
|
216
|
-
* server private key.
|
217
|
-
interceptors : Optional[Sequence[grpc.ServerInterceptor]] (default: None)
|
218
|
-
A list of gRPC interceptors.
|
219
|
-
|
220
|
-
Returns
|
221
|
-
-------
|
222
|
-
server : grpc.Server
|
223
|
-
A non-running instance of a gRPC server.
|
224
|
-
"""
|
225
|
-
# Check if port is in use
|
226
|
-
if is_port_in_use(server_address):
|
227
|
-
sys.exit(f"Port in server address {server_address} is already in use.")
|
228
|
-
|
229
|
-
# Deconstruct tuple into servicer and function
|
230
|
-
servicer, add_servicer_to_server_fn = servicer_and_add_fn
|
231
|
-
|
232
|
-
# Possible options:
|
233
|
-
# https://github.com/grpc/grpc/blob/v1.43.x/include/grpc/impl/codegen/grpc_types.h
|
234
|
-
options = [
|
235
|
-
# Maximum number of concurrent incoming streams to allow on a http2
|
236
|
-
# connection. Int valued.
|
237
|
-
("grpc.max_concurrent_streams", max(100, max_concurrent_workers)),
|
238
|
-
# Maximum message length that the channel can send.
|
239
|
-
# Int valued, bytes. -1 means unlimited.
|
240
|
-
("grpc.max_send_message_length", max_message_length),
|
241
|
-
# Maximum message length that the channel can receive.
|
242
|
-
# Int valued, bytes. -1 means unlimited.
|
243
|
-
("grpc.max_receive_message_length", max_message_length),
|
244
|
-
# The gRPC default for this setting is 7200000 (2 hours). Flower uses a
|
245
|
-
# customized default of 210000 (3 minutes and 30 seconds) to improve
|
246
|
-
# compatibility with popular cloud providers. Mobile Flower clients may
|
247
|
-
# choose to increase this value if their server environment allows
|
248
|
-
# long-running idle TCP connections.
|
249
|
-
("grpc.keepalive_time_ms", keepalive_time_ms),
|
250
|
-
# Setting this to zero will allow sending unlimited keepalive pings in between
|
251
|
-
# sending actual data frames.
|
252
|
-
("grpc.http2.max_pings_without_data", 0),
|
253
|
-
# Is it permissible to send keepalive pings from the client without
|
254
|
-
# any outstanding streams. More explanation here:
|
255
|
-
# https://github.com/adap/flower/pull/2197
|
256
|
-
("grpc.keepalive_permit_without_calls", 0),
|
257
|
-
]
|
258
|
-
|
259
|
-
server = grpc.server(
|
260
|
-
concurrent.futures.ThreadPoolExecutor(max_workers=max_concurrent_workers),
|
261
|
-
# Set the maximum number of concurrent RPCs this server will service before
|
262
|
-
# returning RESOURCE_EXHAUSTED status, or None to indicate no limit.
|
263
|
-
maximum_concurrent_rpcs=max_concurrent_workers,
|
264
|
-
options=options,
|
265
|
-
interceptors=interceptors,
|
266
|
-
)
|
267
|
-
add_servicer_to_server_fn(servicer, server)
|
268
|
-
|
269
|
-
if certificates is not None:
|
270
|
-
if not valid_certificates(certificates):
|
271
|
-
sys.exit(1)
|
272
|
-
|
273
|
-
root_certificate_b, certificate_b, private_key_b = certificates
|
274
|
-
|
275
|
-
server_credentials = grpc.ssl_server_credentials(
|
276
|
-
((private_key_b, certificate_b),),
|
277
|
-
root_certificates=root_certificate_b,
|
278
|
-
# A boolean indicating whether or not to require clients to be
|
279
|
-
# authenticated. May only be True if root_certificates is not None.
|
280
|
-
# We are explicitly setting the current gRPC default to document
|
281
|
-
# the option. For further reference see:
|
282
|
-
# https://grpc.github.io/grpc/python/grpc.html#create-server-credentials
|
283
|
-
require_client_auth=False,
|
284
|
-
)
|
285
|
-
server.add_secure_port(server_address, server_credentials)
|
286
|
-
else:
|
287
|
-
server.add_insecure_port(server_address)
|
288
|
-
|
289
|
-
return server
|