flwr-nightly 1.15.0.dev20250109__py3-none-any.whl → 1.15.0.dev20250110__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flwr/client/app.py +17 -9
- flwr/common/grpc.py +154 -3
- flwr/server/app.py +42 -20
- flwr/server/superlink/driver/serverappio_grpc.py +1 -1
- flwr/server/superlink/fleet/grpc_bidi/grpc_server.py +2 -165
- flwr/server/superlink/simulation/simulationio_grpc.py +1 -1
- flwr/superexec/exec_grpc.py +1 -1
- {flwr_nightly-1.15.0.dev20250109.dist-info → flwr_nightly-1.15.0.dev20250110.dist-info}/METADATA +1 -1
- {flwr_nightly-1.15.0.dev20250109.dist-info → flwr_nightly-1.15.0.dev20250110.dist-info}/RECORD +12 -12
- {flwr_nightly-1.15.0.dev20250109.dist-info → flwr_nightly-1.15.0.dev20250110.dist-info}/LICENSE +0 -0
- {flwr_nightly-1.15.0.dev20250109.dist-info → flwr_nightly-1.15.0.dev20250110.dist-info}/WHEEL +0 -0
- {flwr_nightly-1.15.0.dev20250109.dist-info → flwr_nightly-1.15.0.dev20250110.dist-info}/entry_points.txt +0 -0
flwr/client/app.py
CHANGED
@@ -15,13 +15,14 @@
|
|
15
15
|
"""Flower client app."""
|
16
16
|
|
17
17
|
|
18
|
+
import multiprocessing
|
18
19
|
import signal
|
19
|
-
import subprocess
|
20
20
|
import sys
|
21
21
|
import time
|
22
22
|
from contextlib import AbstractContextManager
|
23
23
|
from dataclasses import dataclass
|
24
24
|
from logging import ERROR, INFO, WARN
|
25
|
+
from os import urandom
|
25
26
|
from pathlib import Path
|
26
27
|
from typing import Callable, Optional, Union, cast
|
27
28
|
|
@@ -33,6 +34,7 @@ from flwr.cli.config_utils import get_fab_metadata
|
|
33
34
|
from flwr.cli.install import install_from_fab
|
34
35
|
from flwr.client.client import Client
|
35
36
|
from flwr.client.client_app import ClientApp, LoadClientAppError
|
37
|
+
from flwr.client.clientapp.app import flwr_clientapp
|
36
38
|
from flwr.client.nodestate.nodestate_factory import NodeStateFactory
|
37
39
|
from flwr.client.typing import ClientFnExt
|
38
40
|
from flwr.common import GRPC_MAX_MESSAGE_LENGTH, Context, EventType, Message, event
|
@@ -53,13 +55,12 @@ from flwr.common.constant import (
|
|
53
55
|
TRANSPORT_TYPES,
|
54
56
|
ErrorCode,
|
55
57
|
)
|
58
|
+
from flwr.common.grpc import generic_create_grpc_server
|
56
59
|
from flwr.common.logger import log, warn_deprecated_feature
|
57
60
|
from flwr.common.message import Error
|
58
61
|
from flwr.common.retry_invoker import RetryInvoker, RetryState, exponential
|
59
62
|
from flwr.common.typing import Fab, Run, RunNotRunningException, UserConfig
|
60
63
|
from flwr.proto.clientappio_pb2_grpc import add_ClientAppIoServicer_to_server
|
61
|
-
from flwr.server.superlink.fleet.grpc_bidi.grpc_server import generic_create_grpc_server
|
62
|
-
from flwr.server.superlink.linkstate.utils import generate_rand_int_from_bytes
|
63
64
|
|
64
65
|
from .clientapp.clientappio_servicer import ClientAppInputs, ClientAppIoServicer
|
65
66
|
from .grpc_adapter_client.connection import grpc_adapter
|
@@ -391,6 +392,7 @@ def start_client_internal(
|
|
391
392
|
run_info_store: Optional[DeprecatedRunInfoStore] = None
|
392
393
|
state_factory = NodeStateFactory()
|
393
394
|
state = state_factory.state()
|
395
|
+
mp_spawn_context = multiprocessing.get_context("spawn")
|
394
396
|
|
395
397
|
runs: dict[int, Run] = {}
|
396
398
|
|
@@ -513,7 +515,7 @@ def start_client_internal(
|
|
513
515
|
# Docker container.
|
514
516
|
|
515
517
|
# Generate SuperNode token
|
516
|
-
token
|
518
|
+
token = int.from_bytes(urandom(RUN_ID_NUM_BYTES), "little")
|
517
519
|
|
518
520
|
# Mode 1: SuperNode starts ClientApp as subprocess
|
519
521
|
start_subprocess = isolation == ISOLATION_MODE_SUBPROCESS
|
@@ -549,12 +551,13 @@ def start_client_internal(
|
|
549
551
|
]
|
550
552
|
command.append("--insecure")
|
551
553
|
|
552
|
-
|
553
|
-
|
554
|
-
|
555
|
-
|
556
|
-
check=True,
|
554
|
+
proc = mp_spawn_context.Process(
|
555
|
+
target=_run_flwr_clientapp,
|
556
|
+
args=(command,),
|
557
|
+
daemon=True,
|
557
558
|
)
|
559
|
+
proc.start()
|
560
|
+
proc.join()
|
558
561
|
else:
|
559
562
|
# Wait for output to become available
|
560
563
|
while not clientappio_servicer.has_outputs():
|
@@ -826,6 +829,11 @@ class _AppStateTracker:
|
|
826
829
|
signal.signal(signal.SIGTERM, signal_handler)
|
827
830
|
|
828
831
|
|
832
|
+
def _run_flwr_clientapp(args: list[str]) -> None:
|
833
|
+
sys.argv = args
|
834
|
+
flwr_clientapp()
|
835
|
+
|
836
|
+
|
829
837
|
def run_clientappio_api_grpc(
|
830
838
|
address: str,
|
831
839
|
certificates: Optional[tuple[bytes, bytes, bytes]],
|
flwr/common/grpc.py
CHANGED
@@ -15,16 +15,26 @@
|
|
15
15
|
"""Utility functions for gRPC."""
|
16
16
|
|
17
17
|
|
18
|
+
import concurrent.futures
|
19
|
+
import sys
|
18
20
|
from collections.abc import Sequence
|
19
|
-
from logging import DEBUG
|
20
|
-
from typing import Optional
|
21
|
+
from logging import DEBUG, ERROR
|
22
|
+
from typing import Any, Callable, Optional
|
21
23
|
|
22
24
|
import grpc
|
23
25
|
|
24
|
-
from
|
26
|
+
from .address import is_port_in_use
|
27
|
+
from .logger import log
|
25
28
|
|
26
29
|
GRPC_MAX_MESSAGE_LENGTH: int = 536_870_912 # == 512 * 1024 * 1024
|
27
30
|
|
31
|
+
INVALID_CERTIFICATES_ERR_MSG = """
|
32
|
+
When setting any of root_certificate, certificate, or private_key,
|
33
|
+
all of them need to be set.
|
34
|
+
"""
|
35
|
+
|
36
|
+
AddServicerToServerFn = Callable[..., Any]
|
37
|
+
|
28
38
|
|
29
39
|
def create_channel(
|
30
40
|
server_address: str,
|
@@ -66,3 +76,144 @@ def create_channel(
|
|
66
76
|
channel = grpc.intercept_channel(channel, interceptors)
|
67
77
|
|
68
78
|
return channel
|
79
|
+
|
80
|
+
|
81
|
+
def valid_certificates(certificates: tuple[bytes, bytes, bytes]) -> bool:
|
82
|
+
"""Validate certificates tuple."""
|
83
|
+
is_valid = (
|
84
|
+
all(isinstance(certificate, bytes) for certificate in certificates)
|
85
|
+
and len(certificates) == 3
|
86
|
+
)
|
87
|
+
|
88
|
+
if not is_valid:
|
89
|
+
log(ERROR, INVALID_CERTIFICATES_ERR_MSG)
|
90
|
+
|
91
|
+
return is_valid
|
92
|
+
|
93
|
+
|
94
|
+
def generic_create_grpc_server( # pylint: disable=too-many-arguments,R0917
|
95
|
+
servicer_and_add_fn: tuple[Any, AddServicerToServerFn],
|
96
|
+
server_address: str,
|
97
|
+
max_concurrent_workers: int = 1000,
|
98
|
+
max_message_length: int = GRPC_MAX_MESSAGE_LENGTH,
|
99
|
+
keepalive_time_ms: int = 210000,
|
100
|
+
certificates: Optional[tuple[bytes, bytes, bytes]] = None,
|
101
|
+
interceptors: Optional[Sequence[grpc.ServerInterceptor]] = None,
|
102
|
+
) -> grpc.Server:
|
103
|
+
"""Create a gRPC server with a single servicer.
|
104
|
+
|
105
|
+
Parameters
|
106
|
+
----------
|
107
|
+
servicer_and_add_fn : tuple
|
108
|
+
A tuple holding a servicer implementation and a matching
|
109
|
+
add_Servicer_to_server function.
|
110
|
+
server_address : str
|
111
|
+
Server address in the form of HOST:PORT e.g. "[::]:8080"
|
112
|
+
max_concurrent_workers : int
|
113
|
+
Maximum number of clients the server can process before returning
|
114
|
+
RESOURCE_EXHAUSTED status (default: 1000)
|
115
|
+
max_message_length : int
|
116
|
+
Maximum message length that the server can send or receive.
|
117
|
+
Int valued in bytes. -1 means unlimited. (default: GRPC_MAX_MESSAGE_LENGTH)
|
118
|
+
keepalive_time_ms : int
|
119
|
+
Flower uses a default gRPC keepalive time of 210000ms (3 minutes 30 seconds)
|
120
|
+
because some cloud providers (for example, Azure) agressively clean up idle
|
121
|
+
TCP connections by terminating them after some time (4 minutes in the case
|
122
|
+
of Azure). Flower does not use application-level keepalive signals and relies
|
123
|
+
on the assumption that the transport layer will fail in cases where the
|
124
|
+
connection is no longer active. `keepalive_time_ms` can be used to customize
|
125
|
+
the keepalive interval for specific environments. The default Flower gRPC
|
126
|
+
keepalive of 210000 ms (3 minutes 30 seconds) ensures that Flower can keep
|
127
|
+
the long running streaming connection alive in most environments. The actual
|
128
|
+
gRPC default of this setting is 7200000 (2 hours), which results in dropped
|
129
|
+
connections in some cloud environments.
|
130
|
+
|
131
|
+
These settings are related to the issue described here:
|
132
|
+
- https://github.com/grpc/proposal/blob/master/A8-client-side-keepalive.md
|
133
|
+
- https://github.com/grpc/grpc/blob/master/doc/keepalive.md
|
134
|
+
- https://grpc.io/docs/guides/performance/
|
135
|
+
|
136
|
+
Mobile Flower clients may choose to increase this value if their server
|
137
|
+
environment allows long-running idle TCP connections.
|
138
|
+
(default: 210000)
|
139
|
+
certificates : Tuple[bytes, bytes, bytes] (default: None)
|
140
|
+
Tuple containing root certificate, server certificate, and private key to
|
141
|
+
start a secure SSL-enabled server. The tuple is expected to have three bytes
|
142
|
+
elements in the following order:
|
143
|
+
|
144
|
+
* CA certificate.
|
145
|
+
* server certificate.
|
146
|
+
* server private key.
|
147
|
+
interceptors : Optional[Sequence[grpc.ServerInterceptor]] (default: None)
|
148
|
+
A list of gRPC interceptors.
|
149
|
+
|
150
|
+
Returns
|
151
|
+
-------
|
152
|
+
server : grpc.Server
|
153
|
+
A non-running instance of a gRPC server.
|
154
|
+
"""
|
155
|
+
# Check if port is in use
|
156
|
+
if is_port_in_use(server_address):
|
157
|
+
sys.exit(f"Port in server address {server_address} is already in use.")
|
158
|
+
|
159
|
+
# Deconstruct tuple into servicer and function
|
160
|
+
servicer, add_servicer_to_server_fn = servicer_and_add_fn
|
161
|
+
|
162
|
+
# Possible options:
|
163
|
+
# https://github.com/grpc/grpc/blob/v1.43.x/include/grpc/impl/codegen/grpc_types.h
|
164
|
+
options = [
|
165
|
+
# Maximum number of concurrent incoming streams to allow on a http2
|
166
|
+
# connection. Int valued.
|
167
|
+
("grpc.max_concurrent_streams", max(100, max_concurrent_workers)),
|
168
|
+
# Maximum message length that the channel can send.
|
169
|
+
# Int valued, bytes. -1 means unlimited.
|
170
|
+
("grpc.max_send_message_length", max_message_length),
|
171
|
+
# Maximum message length that the channel can receive.
|
172
|
+
# Int valued, bytes. -1 means unlimited.
|
173
|
+
("grpc.max_receive_message_length", max_message_length),
|
174
|
+
# The gRPC default for this setting is 7200000 (2 hours). Flower uses a
|
175
|
+
# customized default of 210000 (3 minutes and 30 seconds) to improve
|
176
|
+
# compatibility with popular cloud providers. Mobile Flower clients may
|
177
|
+
# choose to increase this value if their server environment allows
|
178
|
+
# long-running idle TCP connections.
|
179
|
+
("grpc.keepalive_time_ms", keepalive_time_ms),
|
180
|
+
# Setting this to zero will allow sending unlimited keepalive pings in between
|
181
|
+
# sending actual data frames.
|
182
|
+
("grpc.http2.max_pings_without_data", 0),
|
183
|
+
# Is it permissible to send keepalive pings from the client without
|
184
|
+
# any outstanding streams. More explanation here:
|
185
|
+
# https://github.com/adap/flower/pull/2197
|
186
|
+
("grpc.keepalive_permit_without_calls", 0),
|
187
|
+
]
|
188
|
+
|
189
|
+
server = grpc.server(
|
190
|
+
concurrent.futures.ThreadPoolExecutor(max_workers=max_concurrent_workers),
|
191
|
+
# Set the maximum number of concurrent RPCs this server will service before
|
192
|
+
# returning RESOURCE_EXHAUSTED status, or None to indicate no limit.
|
193
|
+
maximum_concurrent_rpcs=max_concurrent_workers,
|
194
|
+
options=options,
|
195
|
+
interceptors=interceptors,
|
196
|
+
)
|
197
|
+
add_servicer_to_server_fn(servicer, server)
|
198
|
+
|
199
|
+
if certificates is not None:
|
200
|
+
if not valid_certificates(certificates):
|
201
|
+
sys.exit(1)
|
202
|
+
|
203
|
+
root_certificate_b, certificate_b, private_key_b = certificates
|
204
|
+
|
205
|
+
server_credentials = grpc.ssl_server_credentials(
|
206
|
+
((private_key_b, certificate_b),),
|
207
|
+
root_certificates=root_certificate_b,
|
208
|
+
# A boolean indicating whether or not to require clients to be
|
209
|
+
# authenticated. May only be True if root_certificates is not None.
|
210
|
+
# We are explicitly setting the current gRPC default to document
|
211
|
+
# the option. For further reference see:
|
212
|
+
# https://grpc.github.io/grpc/python/grpc.html#create-server-credentials
|
213
|
+
require_client_auth=False,
|
214
|
+
)
|
215
|
+
server.add_secure_port(server_address, server_credentials)
|
216
|
+
else:
|
217
|
+
server.add_insecure_port(server_address)
|
218
|
+
|
219
|
+
return server
|
flwr/server/app.py
CHANGED
@@ -18,7 +18,8 @@
|
|
18
18
|
import argparse
|
19
19
|
import csv
|
20
20
|
import importlib.util
|
21
|
-
import
|
21
|
+
import multiprocessing
|
22
|
+
import multiprocessing.context
|
22
23
|
import sys
|
23
24
|
import threading
|
24
25
|
from collections.abc import Sequence
|
@@ -59,6 +60,7 @@ from flwr.common.constant import (
|
|
59
60
|
TRANSPORT_TYPE_REST,
|
60
61
|
)
|
61
62
|
from flwr.common.exit_handlers import register_exit_handlers
|
63
|
+
from flwr.common.grpc import generic_create_grpc_server
|
62
64
|
from flwr.common.logger import log, warn_deprecated_feature
|
63
65
|
from flwr.common.secure_aggregation.crypto.symmetric_encryption import (
|
64
66
|
private_key_to_bytes,
|
@@ -68,6 +70,8 @@ from flwr.proto.fleet_pb2_grpc import ( # pylint: disable=E0611
|
|
68
70
|
add_FleetServicer_to_server,
|
69
71
|
)
|
70
72
|
from flwr.proto.grpcadapter_pb2_grpc import add_GrpcAdapterServicer_to_server
|
73
|
+
from flwr.server.serverapp.app import flwr_serverapp
|
74
|
+
from flwr.simulation.app import flwr_simulation
|
71
75
|
from flwr.superexec.app import load_executor
|
72
76
|
from flwr.superexec.exec_grpc import run_exec_api_grpc
|
73
77
|
|
@@ -79,10 +83,7 @@ from .strategy import Strategy
|
|
79
83
|
from .superlink.driver.serverappio_grpc import run_serverappio_api_grpc
|
80
84
|
from .superlink.ffs.ffs_factory import FfsFactory
|
81
85
|
from .superlink.fleet.grpc_adapter.grpc_adapter_servicer import GrpcAdapterServicer
|
82
|
-
from .superlink.fleet.grpc_bidi.grpc_server import
|
83
|
-
generic_create_grpc_server,
|
84
|
-
start_grpc_server,
|
85
|
-
)
|
86
|
+
from .superlink.fleet.grpc_bidi.grpc_server import start_grpc_server
|
86
87
|
from .superlink.fleet.grpc_rere.fleet_servicer import FleetServicer
|
87
88
|
from .superlink.fleet.grpc_rere.server_interceptor import AuthenticateServerInterceptor
|
88
89
|
from .superlink.linkstate import LinkStateFactory
|
@@ -292,7 +293,7 @@ def run_superlink() -> None:
|
|
292
293
|
# Determine Exec plugin
|
293
294
|
# If simulation is used, don't start ServerAppIo and Fleet APIs
|
294
295
|
sim_exec = executor.__class__.__qualname__ == "SimulationEngine"
|
295
|
-
bckg_threads = []
|
296
|
+
bckg_threads: list[threading.Thread] = []
|
296
297
|
|
297
298
|
if sim_exec:
|
298
299
|
simulationio_server: grpc.Server = run_simulationio_api_grpc(
|
@@ -360,6 +361,7 @@ def run_superlink() -> None:
|
|
360
361
|
ffs_factory,
|
361
362
|
num_workers,
|
362
363
|
),
|
364
|
+
daemon=True,
|
363
365
|
)
|
364
366
|
fleet_thread.start()
|
365
367
|
bckg_threads.append(fleet_thread)
|
@@ -426,6 +428,7 @@ def run_superlink() -> None:
|
|
426
428
|
address,
|
427
429
|
cmd,
|
428
430
|
),
|
431
|
+
daemon=True,
|
429
432
|
)
|
430
433
|
scheduler_th.start()
|
431
434
|
bckg_threads.append(scheduler_th)
|
@@ -434,16 +437,24 @@ def run_superlink() -> None:
|
|
434
437
|
register_exit_handlers(
|
435
438
|
event_type=EventType.RUN_SUPERLINK_LEAVE,
|
436
439
|
grpc_servers=grpc_servers,
|
437
|
-
bckg_threads=bckg_threads,
|
438
440
|
)
|
439
441
|
|
440
|
-
# Block
|
441
|
-
while
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
442
|
+
# Block until a thread exits prematurely
|
443
|
+
while all(thread.is_alive() for thread in bckg_threads):
|
444
|
+
sleep(0.1)
|
445
|
+
|
446
|
+
# Exit if any thread has exited prematurely
|
447
|
+
sys.exit(1)
|
448
|
+
|
449
|
+
|
450
|
+
def _run_flwr_command(args: list[str]) -> None:
|
451
|
+
sys.argv = args
|
452
|
+
if args[0] == "flwr-serverapp":
|
453
|
+
flwr_serverapp()
|
454
|
+
elif args[0] == "flwr-simulation":
|
455
|
+
flwr_simulation()
|
456
|
+
else:
|
457
|
+
raise ValueError(f"Unknown command: {args[0]}")
|
447
458
|
|
448
459
|
|
449
460
|
def _flwr_scheduler(
|
@@ -453,15 +464,18 @@ def _flwr_scheduler(
|
|
453
464
|
cmd: str,
|
454
465
|
) -> None:
|
455
466
|
log(DEBUG, "Started %s scheduler thread.", cmd)
|
456
|
-
|
457
467
|
state = state_factory.state()
|
468
|
+
run_id_to_proc: dict[int, multiprocessing.context.SpawnProcess] = {}
|
469
|
+
|
470
|
+
# Use the "spawn" start method for multiprocessing.
|
471
|
+
mp_spawn_context = multiprocessing.get_context("spawn")
|
458
472
|
|
459
473
|
# Periodically check for a pending run in the LinkState
|
460
474
|
while True:
|
461
|
-
sleep(
|
475
|
+
sleep(0.1)
|
462
476
|
pending_run_id = state.get_pending_run_id()
|
463
477
|
|
464
|
-
if pending_run_id:
|
478
|
+
if pending_run_id and pending_run_id not in run_id_to_proc:
|
465
479
|
|
466
480
|
log(
|
467
481
|
INFO,
|
@@ -478,10 +492,18 @@ def _flwr_scheduler(
|
|
478
492
|
"--insecure",
|
479
493
|
]
|
480
494
|
|
481
|
-
|
482
|
-
command,
|
483
|
-
text=True,
|
495
|
+
proc = mp_spawn_context.Process(
|
496
|
+
target=_run_flwr_command, args=(command,), daemon=True
|
484
497
|
)
|
498
|
+
proc.start()
|
499
|
+
|
500
|
+
# Store the process
|
501
|
+
run_id_to_proc[pending_run_id] = proc
|
502
|
+
|
503
|
+
# Clean up finished processes
|
504
|
+
for run_id, proc in list(run_id_to_proc.items()):
|
505
|
+
if not proc.is_alive():
|
506
|
+
del run_id_to_proc[run_id]
|
485
507
|
|
486
508
|
|
487
509
|
def _format_address(address: str) -> tuple[str, str, int]:
|
@@ -21,6 +21,7 @@ from typing import Optional
|
|
21
21
|
import grpc
|
22
22
|
|
23
23
|
from flwr.common import GRPC_MAX_MESSAGE_LENGTH
|
24
|
+
from flwr.common.grpc import generic_create_grpc_server
|
24
25
|
from flwr.common.logger import log
|
25
26
|
from flwr.proto.serverappio_pb2_grpc import ( # pylint: disable=E0611
|
26
27
|
add_ServerAppIoServicer_to_server,
|
@@ -28,7 +29,6 @@ from flwr.proto.serverappio_pb2_grpc import ( # pylint: disable=E0611
|
|
28
29
|
from flwr.server.superlink.ffs.ffs_factory import FfsFactory
|
29
30
|
from flwr.server.superlink.linkstate import LinkStateFactory
|
30
31
|
|
31
|
-
from ..fleet.grpc_bidi.grpc_server import generic_create_grpc_server
|
32
32
|
from .serverappio_servicer import ServerAppIoServicer
|
33
33
|
|
34
34
|
|
@@ -15,49 +15,19 @@
|
|
15
15
|
"""Implements utility function to create a gRPC server."""
|
16
16
|
|
17
17
|
|
18
|
-
import
|
19
|
-
import sys
|
20
|
-
from collections.abc import Sequence
|
21
|
-
from logging import ERROR
|
22
|
-
from typing import Any, Callable, Optional, Union
|
18
|
+
from typing import Optional
|
23
19
|
|
24
20
|
import grpc
|
25
21
|
|
26
22
|
from flwr.common import GRPC_MAX_MESSAGE_LENGTH
|
27
|
-
from flwr.common.
|
28
|
-
from flwr.common.logger import log
|
23
|
+
from flwr.common.grpc import generic_create_grpc_server
|
29
24
|
from flwr.proto.transport_pb2_grpc import ( # pylint: disable=E0611
|
30
25
|
add_FlowerServiceServicer_to_server,
|
31
26
|
)
|
32
27
|
from flwr.server.client_manager import ClientManager
|
33
|
-
from flwr.server.superlink.driver.serverappio_servicer import ServerAppIoServicer
|
34
|
-
from flwr.server.superlink.fleet.grpc_adapter.grpc_adapter_servicer import (
|
35
|
-
GrpcAdapterServicer,
|
36
|
-
)
|
37
28
|
from flwr.server.superlink.fleet.grpc_bidi.flower_service_servicer import (
|
38
29
|
FlowerServiceServicer,
|
39
30
|
)
|
40
|
-
from flwr.server.superlink.fleet.grpc_rere.fleet_servicer import FleetServicer
|
41
|
-
|
42
|
-
INVALID_CERTIFICATES_ERR_MSG = """
|
43
|
-
When setting any of root_certificate, certificate, or private_key,
|
44
|
-
all of them need to be set.
|
45
|
-
"""
|
46
|
-
|
47
|
-
AddServicerToServerFn = Callable[..., Any]
|
48
|
-
|
49
|
-
|
50
|
-
def valid_certificates(certificates: tuple[bytes, bytes, bytes]) -> bool:
|
51
|
-
"""Validate certificates tuple."""
|
52
|
-
is_valid = (
|
53
|
-
all(isinstance(certificate, bytes) for certificate in certificates)
|
54
|
-
and len(certificates) == 3
|
55
|
-
)
|
56
|
-
|
57
|
-
if not is_valid:
|
58
|
-
log(ERROR, INVALID_CERTIFICATES_ERR_MSG)
|
59
|
-
|
60
|
-
return is_valid
|
61
31
|
|
62
32
|
|
63
33
|
def start_grpc_server( # pylint: disable=too-many-arguments,R0917
|
@@ -154,136 +124,3 @@ def start_grpc_server( # pylint: disable=too-many-arguments,R0917
|
|
154
124
|
server.start()
|
155
125
|
|
156
126
|
return server
|
157
|
-
|
158
|
-
|
159
|
-
def generic_create_grpc_server( # pylint: disable=too-many-arguments,R0917
|
160
|
-
servicer_and_add_fn: Union[
|
161
|
-
tuple[FleetServicer, AddServicerToServerFn],
|
162
|
-
tuple[GrpcAdapterServicer, AddServicerToServerFn],
|
163
|
-
tuple[FlowerServiceServicer, AddServicerToServerFn],
|
164
|
-
tuple[ServerAppIoServicer, AddServicerToServerFn],
|
165
|
-
],
|
166
|
-
server_address: str,
|
167
|
-
max_concurrent_workers: int = 1000,
|
168
|
-
max_message_length: int = GRPC_MAX_MESSAGE_LENGTH,
|
169
|
-
keepalive_time_ms: int = 210000,
|
170
|
-
certificates: Optional[tuple[bytes, bytes, bytes]] = None,
|
171
|
-
interceptors: Optional[Sequence[grpc.ServerInterceptor]] = None,
|
172
|
-
) -> grpc.Server:
|
173
|
-
"""Create a gRPC server with a single servicer.
|
174
|
-
|
175
|
-
Parameters
|
176
|
-
----------
|
177
|
-
servicer_and_add_fn : tuple
|
178
|
-
A tuple holding a servicer implementation and a matching
|
179
|
-
add_Servicer_to_server function.
|
180
|
-
server_address : str
|
181
|
-
Server address in the form of HOST:PORT e.g. "[::]:8080"
|
182
|
-
max_concurrent_workers : int
|
183
|
-
Maximum number of clients the server can process before returning
|
184
|
-
RESOURCE_EXHAUSTED status (default: 1000)
|
185
|
-
max_message_length : int
|
186
|
-
Maximum message length that the server can send or receive.
|
187
|
-
Int valued in bytes. -1 means unlimited. (default: GRPC_MAX_MESSAGE_LENGTH)
|
188
|
-
keepalive_time_ms : int
|
189
|
-
Flower uses a default gRPC keepalive time of 210000ms (3 minutes 30 seconds)
|
190
|
-
because some cloud providers (for example, Azure) agressively clean up idle
|
191
|
-
TCP connections by terminating them after some time (4 minutes in the case
|
192
|
-
of Azure). Flower does not use application-level keepalive signals and relies
|
193
|
-
on the assumption that the transport layer will fail in cases where the
|
194
|
-
connection is no longer active. `keepalive_time_ms` can be used to customize
|
195
|
-
the keepalive interval for specific environments. The default Flower gRPC
|
196
|
-
keepalive of 210000 ms (3 minutes 30 seconds) ensures that Flower can keep
|
197
|
-
the long running streaming connection alive in most environments. The actual
|
198
|
-
gRPC default of this setting is 7200000 (2 hours), which results in dropped
|
199
|
-
connections in some cloud environments.
|
200
|
-
|
201
|
-
These settings are related to the issue described here:
|
202
|
-
- https://github.com/grpc/proposal/blob/master/A8-client-side-keepalive.md
|
203
|
-
- https://github.com/grpc/grpc/blob/master/doc/keepalive.md
|
204
|
-
- https://grpc.io/docs/guides/performance/
|
205
|
-
|
206
|
-
Mobile Flower clients may choose to increase this value if their server
|
207
|
-
environment allows long-running idle TCP connections.
|
208
|
-
(default: 210000)
|
209
|
-
certificates : Tuple[bytes, bytes, bytes] (default: None)
|
210
|
-
Tuple containing root certificate, server certificate, and private key to
|
211
|
-
start a secure SSL-enabled server. The tuple is expected to have three bytes
|
212
|
-
elements in the following order:
|
213
|
-
|
214
|
-
* CA certificate.
|
215
|
-
* server certificate.
|
216
|
-
* server private key.
|
217
|
-
interceptors : Optional[Sequence[grpc.ServerInterceptor]] (default: None)
|
218
|
-
A list of gRPC interceptors.
|
219
|
-
|
220
|
-
Returns
|
221
|
-
-------
|
222
|
-
server : grpc.Server
|
223
|
-
A non-running instance of a gRPC server.
|
224
|
-
"""
|
225
|
-
# Check if port is in use
|
226
|
-
if is_port_in_use(server_address):
|
227
|
-
sys.exit(f"Port in server address {server_address} is already in use.")
|
228
|
-
|
229
|
-
# Deconstruct tuple into servicer and function
|
230
|
-
servicer, add_servicer_to_server_fn = servicer_and_add_fn
|
231
|
-
|
232
|
-
# Possible options:
|
233
|
-
# https://github.com/grpc/grpc/blob/v1.43.x/include/grpc/impl/codegen/grpc_types.h
|
234
|
-
options = [
|
235
|
-
# Maximum number of concurrent incoming streams to allow on a http2
|
236
|
-
# connection. Int valued.
|
237
|
-
("grpc.max_concurrent_streams", max(100, max_concurrent_workers)),
|
238
|
-
# Maximum message length that the channel can send.
|
239
|
-
# Int valued, bytes. -1 means unlimited.
|
240
|
-
("grpc.max_send_message_length", max_message_length),
|
241
|
-
# Maximum message length that the channel can receive.
|
242
|
-
# Int valued, bytes. -1 means unlimited.
|
243
|
-
("grpc.max_receive_message_length", max_message_length),
|
244
|
-
# The gRPC default for this setting is 7200000 (2 hours). Flower uses a
|
245
|
-
# customized default of 210000 (3 minutes and 30 seconds) to improve
|
246
|
-
# compatibility with popular cloud providers. Mobile Flower clients may
|
247
|
-
# choose to increase this value if their server environment allows
|
248
|
-
# long-running idle TCP connections.
|
249
|
-
("grpc.keepalive_time_ms", keepalive_time_ms),
|
250
|
-
# Setting this to zero will allow sending unlimited keepalive pings in between
|
251
|
-
# sending actual data frames.
|
252
|
-
("grpc.http2.max_pings_without_data", 0),
|
253
|
-
# Is it permissible to send keepalive pings from the client without
|
254
|
-
# any outstanding streams. More explanation here:
|
255
|
-
# https://github.com/adap/flower/pull/2197
|
256
|
-
("grpc.keepalive_permit_without_calls", 0),
|
257
|
-
]
|
258
|
-
|
259
|
-
server = grpc.server(
|
260
|
-
concurrent.futures.ThreadPoolExecutor(max_workers=max_concurrent_workers),
|
261
|
-
# Set the maximum number of concurrent RPCs this server will service before
|
262
|
-
# returning RESOURCE_EXHAUSTED status, or None to indicate no limit.
|
263
|
-
maximum_concurrent_rpcs=max_concurrent_workers,
|
264
|
-
options=options,
|
265
|
-
interceptors=interceptors,
|
266
|
-
)
|
267
|
-
add_servicer_to_server_fn(servicer, server)
|
268
|
-
|
269
|
-
if certificates is not None:
|
270
|
-
if not valid_certificates(certificates):
|
271
|
-
sys.exit(1)
|
272
|
-
|
273
|
-
root_certificate_b, certificate_b, private_key_b = certificates
|
274
|
-
|
275
|
-
server_credentials = grpc.ssl_server_credentials(
|
276
|
-
((private_key_b, certificate_b),),
|
277
|
-
root_certificates=root_certificate_b,
|
278
|
-
# A boolean indicating whether or not to require clients to be
|
279
|
-
# authenticated. May only be True if root_certificates is not None.
|
280
|
-
# We are explicitly setting the current gRPC default to document
|
281
|
-
# the option. For further reference see:
|
282
|
-
# https://grpc.github.io/grpc/python/grpc.html#create-server-credentials
|
283
|
-
require_client_auth=False,
|
284
|
-
)
|
285
|
-
server.add_secure_port(server_address, server_credentials)
|
286
|
-
else:
|
287
|
-
server.add_insecure_port(server_address)
|
288
|
-
|
289
|
-
return server
|
@@ -21,6 +21,7 @@ from typing import Optional
|
|
21
21
|
import grpc
|
22
22
|
|
23
23
|
from flwr.common import GRPC_MAX_MESSAGE_LENGTH
|
24
|
+
from flwr.common.grpc import generic_create_grpc_server
|
24
25
|
from flwr.common.logger import log
|
25
26
|
from flwr.proto.simulationio_pb2_grpc import ( # pylint: disable=E0611
|
26
27
|
add_SimulationIoServicer_to_server,
|
@@ -28,7 +29,6 @@ from flwr.proto.simulationio_pb2_grpc import ( # pylint: disable=E0611
|
|
28
29
|
from flwr.server.superlink.ffs.ffs_factory import FfsFactory
|
29
30
|
from flwr.server.superlink.linkstate import LinkStateFactory
|
30
31
|
|
31
|
-
from ..fleet.grpc_bidi.grpc_server import generic_create_grpc_server
|
32
32
|
from .simulationio_servicer import SimulationIoServicer
|
33
33
|
|
34
34
|
|
flwr/superexec/exec_grpc.py
CHANGED
@@ -23,11 +23,11 @@ import grpc
|
|
23
23
|
|
24
24
|
from flwr.common import GRPC_MAX_MESSAGE_LENGTH
|
25
25
|
from flwr.common.auth_plugin import ExecAuthPlugin
|
26
|
+
from flwr.common.grpc import generic_create_grpc_server
|
26
27
|
from flwr.common.logger import log
|
27
28
|
from flwr.common.typing import UserConfig
|
28
29
|
from flwr.proto.exec_pb2_grpc import add_ExecServicer_to_server
|
29
30
|
from flwr.server.superlink.ffs.ffs_factory import FfsFactory
|
30
|
-
from flwr.server.superlink.fleet.grpc_bidi.grpc_server import generic_create_grpc_server
|
31
31
|
from flwr.server.superlink.linkstate import LinkStateFactory
|
32
32
|
from flwr.superexec.exec_user_auth_interceptor import ExecUserAuthInterceptor
|
33
33
|
|
{flwr_nightly-1.15.0.dev20250109.dist-info → flwr_nightly-1.15.0.dev20250110.dist-info}/RECORD
RENAMED
@@ -69,7 +69,7 @@ flwr/cli/run/run.py,sha256=BvpjYyUvDhVMvO5cG711ihtdeSbls9p8zVAuFGETLA8,7893
|
|
69
69
|
flwr/cli/stop.py,sha256=1T9RNRCH8dxjmBT38hFtKAWY9Gb7RMCMCML7kex9WzE,4613
|
70
70
|
flwr/cli/utils.py,sha256=RXozds-T7HN8yH0mMj67q-0YktdNNm83N9Ptu_pdhsc,10604
|
71
71
|
flwr/client/__init__.py,sha256=DGDoO0AEAfz-0CUFmLdyUUweAS64-07AOnmDfWUefK4,1192
|
72
|
-
flwr/client/app.py,sha256=
|
72
|
+
flwr/client/app.py,sha256=IboXEVp3TeCF_3KErGI7zHg0C7ZAwi4evcf9PLsCFm8,34987
|
73
73
|
flwr/client/client.py,sha256=8o58nd9o6ZFcMIaVYPGcV4MSjBG4H0oFgWiv8ZEO3oA,7895
|
74
74
|
flwr/client/client_app.py,sha256=cTig-N00YzTucbo9zNi6I21J8PlbflU_8J_f5CI-Wpw,10390
|
75
75
|
flwr/client/clientapp/__init__.py,sha256=kZqChGnTChQ1WGSUkIlW2S5bc0d0mzDubCAmZUGRpEY,800
|
@@ -121,7 +121,7 @@ flwr/common/differential_privacy.py,sha256=XwcJ3rWr8S8BZUocc76vLSJAXIf6OHnWkBV6-
|
|
121
121
|
flwr/common/differential_privacy_constants.py,sha256=c7b7tqgvT7yMK0XN9ndiTBs4mQf6d3qk6K7KBZGlV4Q,1074
|
122
122
|
flwr/common/dp.py,sha256=vddkvyjV2FhRoN4VuU2LeAM1UBn7dQB8_W-Qdiveal8,1978
|
123
123
|
flwr/common/exit_handlers.py,sha256=MracJaBeoCOC7TaXK9zCJQxhrMSx9ZtczK237qvhBpU,2806
|
124
|
-
flwr/common/grpc.py,sha256=
|
124
|
+
flwr/common/grpc.py,sha256=amBroQiRdRrczDAGbEZjaYsVoeH_52pyfyOvTSvIF_c,9362
|
125
125
|
flwr/common/logger.py,sha256=qwiOc9N_Dqh-NlxtENcMa-dCPqint20ZLuWEvnAEwHU,12323
|
126
126
|
flwr/common/message.py,sha256=Zv4ID2BLQsbff0F03DI_MeFoHbSqVZAdDD9NcKYv6Zo,13832
|
127
127
|
flwr/common/object_ref.py,sha256=fIXf8aP5mG6Nuni7dvcKK5Di3zRfRWGs4ljvqIXplds,10115
|
@@ -211,7 +211,7 @@ flwr/proto/transport_pb2_grpc.py,sha256=vLN3EHtx2aEEMCO4f1Upu-l27BPzd3-5pV-u8wPc
|
|
211
211
|
flwr/proto/transport_pb2_grpc.pyi,sha256=AGXf8RiIiW2J5IKMlm_3qT3AzcDa4F3P5IqUjve_esA,766
|
212
212
|
flwr/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
213
213
|
flwr/server/__init__.py,sha256=cEg1oecBu4cKB69iJCqWEylC8b5XW47bl7rQiJsdTvM,1528
|
214
|
-
flwr/server/app.py,sha256=
|
214
|
+
flwr/server/app.py,sha256=uhX8FE2II2vv-34ENUDVsp1HOZXlS-8kBOfd7TOXn0E,31799
|
215
215
|
flwr/server/client_manager.py,sha256=7Ese0tgrH-i-ms363feYZJKwB8gWnXSmg_hYF2Bju4U,6227
|
216
216
|
flwr/server/client_proxy.py,sha256=4G-oTwhb45sfWLx2uZdcXD98IZwdTS6F88xe3akCdUg,2399
|
217
217
|
flwr/server/compat/__init__.py,sha256=VxnJtJyOjNFQXMNi9hIuzNlZM5n0Hj1p3aq_Pm2udw4,892
|
@@ -258,7 +258,7 @@ flwr/server/strategy/qfedavg.py,sha256=2ijNNc2vVODWLAaoYo9PCoaFvlanq0lbJ7I7Albdu
|
|
258
258
|
flwr/server/strategy/strategy.py,sha256=cXapkD5uDrt5C-RbmWDn9FLoap3Q41i7GKvbmfbCKtk,7524
|
259
259
|
flwr/server/superlink/__init__.py,sha256=8tHYCfodUlRD8PCP9fHgvu8cz5N31A2QoRVL0jDJ15E,707
|
260
260
|
flwr/server/superlink/driver/__init__.py,sha256=5soEK5QSvxNjmJQ-CGTWROc4alSAeU0e9Ad9RDhsd3E,717
|
261
|
-
flwr/server/superlink/driver/serverappio_grpc.py,sha256=
|
261
|
+
flwr/server/superlink/driver/serverappio_grpc.py,sha256=UzHwo6qYZMeOhr7nn1iZbcyDSmwvnq_kpYH0mEAndW0,2173
|
262
262
|
flwr/server/superlink/driver/serverappio_servicer.py,sha256=4QEwqm_pew1iVbtEZuOWLhfgbbw4LIUwFjOuGBlPJZY,12747
|
263
263
|
flwr/server/superlink/ffs/__init__.py,sha256=FAY-zShcfPmOxosok2QyT6hTNMNctG8cH9s_nIl8jkI,840
|
264
264
|
flwr/server/superlink/ffs/disk_ffs.py,sha256=n_Ah0sQwXGVQ9wj5965nLjdkQQbpoHCljjXKFnwftsU,3297
|
@@ -271,7 +271,7 @@ flwr/server/superlink/fleet/grpc_bidi/__init__.py,sha256=dkSKQMuMTYh1qSnuN87cAPv
|
|
271
271
|
flwr/server/superlink/fleet/grpc_bidi/flower_service_servicer.py,sha256=ud08wi9j8OYRYVTIioL1xenOgrEbtS7afyr8MnQEk4I,6021
|
272
272
|
flwr/server/superlink/fleet/grpc_bidi/grpc_bridge.py,sha256=JkAH_nIZaqe_9kntrg26od_jaz5XdLFuvNMgGu8xk9Q,6485
|
273
273
|
flwr/server/superlink/fleet/grpc_bidi/grpc_client_proxy.py,sha256=h3EhqgelegVC4EjOXH5birmAnMoCBJcP7jpHYCnHZPk,4887
|
274
|
-
flwr/server/superlink/fleet/grpc_bidi/grpc_server.py,sha256=
|
274
|
+
flwr/server/superlink/fleet/grpc_bidi/grpc_server.py,sha256=mxPxyEF0IW0vV41Bqk1zfKOdRDEvXPwzJyMiRMg7nTI,5173
|
275
275
|
flwr/server/superlink/fleet/grpc_rere/__init__.py,sha256=j2hyC342am-_Hgp1g80Y3fGDzfTI6n8QOOn2PyWf4eg,758
|
276
276
|
flwr/server/superlink/fleet/grpc_rere/fleet_servicer.py,sha256=PyZP78gHU_RIRVSlPjyAvKG6UOLSilopkT6zoC_6ASc,5881
|
277
277
|
flwr/server/superlink/fleet/grpc_rere/server_interceptor.py,sha256=SKa1wfBNxJL6vGBUbbTgUylY3ZHJMyBSZRC1J6PBvyY,8342
|
@@ -291,7 +291,7 @@ flwr/server/superlink/linkstate/linkstate_factory.py,sha256=ISSMjDlwuN7swxjOeYlT
|
|
291
291
|
flwr/server/superlink/linkstate/sqlite_linkstate.py,sha256=lLvVAscwVioqzj4uj0WfDUyoq-L7Nr-kMKCdaGPI0Z8,44050
|
292
292
|
flwr/server/superlink/linkstate/utils.py,sha256=d5uqqIOCKfd54X8CFNfUr3AWqPLpgmzsC_RagRwFugM,13321
|
293
293
|
flwr/server/superlink/simulation/__init__.py,sha256=mg-oapC9dkzEfjXPQFior5lpWj4g9kwbLovptyYM_g0,718
|
294
|
-
flwr/server/superlink/simulation/simulationio_grpc.py,sha256=
|
294
|
+
flwr/server/superlink/simulation/simulationio_grpc.py,sha256=8aUrZZLdvprKUfLLqFID4aItus9beU6m1qLQYIPB7k0,2224
|
295
295
|
flwr/server/superlink/simulation/simulationio_servicer.py,sha256=J_TmdqM-Bxgp-iPEI3tvCuBpykw1UX0FouMQalEYAF4,6907
|
296
296
|
flwr/server/superlink/utils.py,sha256=KVb3K_g2vYfu9TnftcN0ewmev133WZcjuEePMm8d7GE,2137
|
297
297
|
flwr/server/typing.py,sha256=5kaRLZuxTEse9A0g7aVna2VhYxU3wTq1f3d3mtw7kXs,1019
|
@@ -316,13 +316,13 @@ flwr/simulation/simulationio_connection.py,sha256=8aAh6MKQkQPMSnWEqA5vua_QzZtoMx
|
|
316
316
|
flwr/superexec/__init__.py,sha256=fcj366jh4RFby_vDwLroU4kepzqbnJgseZD_jUr_Mko,715
|
317
317
|
flwr/superexec/app.py,sha256=Z6kYHWd62YL0Q4YKyCAbt_BcefNfbKH6V-jCC-1NkZM,1842
|
318
318
|
flwr/superexec/deployment.py,sha256=wZ9G42gGS91knfplswh95MnQ83Fzu-rs6wcuNgDmmvY,6735
|
319
|
-
flwr/superexec/exec_grpc.py,sha256=
|
319
|
+
flwr/superexec/exec_grpc.py,sha256=ttA9qoZzSLF0Mfk1L4hzOkMSNuj5rR58_kKBYwcyrAg,2864
|
320
320
|
flwr/superexec/exec_servicer.py,sha256=X10ILT-AoGMrB3IgI2mBe9i-QcIVUAl9bucuqVOPYkU,8341
|
321
321
|
flwr/superexec/exec_user_auth_interceptor.py,sha256=K06OU-l4LnYhTDg071hGJuOaQWEJbZsYi5qxUmmtiG0,3704
|
322
322
|
flwr/superexec/executor.py,sha256=_B55WW2TD1fBINpabSSDRenVHXYmvlfhv-k8hJKU4lQ,3115
|
323
323
|
flwr/superexec/simulation.py,sha256=WQDon15oqpMopAZnwRZoTICYCfHqtkvFSqiTQ2hLD_g,4088
|
324
|
-
flwr_nightly-1.15.0.
|
325
|
-
flwr_nightly-1.15.0.
|
326
|
-
flwr_nightly-1.15.0.
|
327
|
-
flwr_nightly-1.15.0.
|
328
|
-
flwr_nightly-1.15.0.
|
324
|
+
flwr_nightly-1.15.0.dev20250110.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
|
325
|
+
flwr_nightly-1.15.0.dev20250110.dist-info/METADATA,sha256=-RWddYXmAKXhM81hEu_pDSDREXGhGS1sCqEHdBueQwc,15882
|
326
|
+
flwr_nightly-1.15.0.dev20250110.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
|
327
|
+
flwr_nightly-1.15.0.dev20250110.dist-info/entry_points.txt,sha256=JlNxX3qhaV18_2yj5a3kJW1ESxm31cal9iS_N_pf1Rk,538
|
328
|
+
flwr_nightly-1.15.0.dev20250110.dist-info/RECORD,,
|
{flwr_nightly-1.15.0.dev20250109.dist-info → flwr_nightly-1.15.0.dev20250110.dist-info}/LICENSE
RENAMED
File without changes
|
{flwr_nightly-1.15.0.dev20250109.dist-info → flwr_nightly-1.15.0.dev20250110.dist-info}/WHEEL
RENAMED
File without changes
|
File without changes
|