flwr 1.12.0__py3-none-any.whl → 1.13.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flwr/cli/app.py +2 -0
- flwr/cli/build.py +2 -2
- flwr/cli/config_utils.py +97 -0
- flwr/cli/install.py +0 -16
- flwr/cli/log.py +63 -97
- flwr/cli/ls.py +228 -0
- flwr/cli/new/new.py +23 -13
- flwr/cli/new/templates/app/README.md.tpl +11 -0
- flwr/cli/new/templates/app/code/flwr_tune/dataset.py.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.baseline.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl +2 -1
- flwr/cli/new/templates/app/pyproject.huggingface.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.jax.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.mlx.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.numpy.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.sklearn.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl +2 -2
- flwr/cli/run/run.py +37 -89
- flwr/client/app.py +73 -34
- flwr/client/clientapp/app.py +58 -37
- flwr/client/grpc_rere_client/connection.py +7 -12
- flwr/client/nodestate/__init__.py +25 -0
- flwr/client/nodestate/in_memory_nodestate.py +38 -0
- flwr/client/nodestate/nodestate.py +30 -0
- flwr/client/nodestate/nodestate_factory.py +37 -0
- flwr/client/rest_client/connection.py +4 -14
- flwr/client/{node_state.py → run_info_store.py} +4 -3
- flwr/client/supernode/app.py +34 -58
- flwr/common/args.py +152 -0
- flwr/common/config.py +10 -0
- flwr/common/constant.py +59 -7
- flwr/common/context.py +9 -4
- flwr/common/date.py +21 -3
- flwr/common/grpc.py +4 -1
- flwr/common/logger.py +108 -1
- flwr/common/object_ref.py +47 -16
- flwr/common/serde.py +34 -0
- flwr/common/telemetry.py +0 -6
- flwr/common/typing.py +32 -2
- flwr/proto/exec_pb2.py +23 -17
- flwr/proto/exec_pb2.pyi +58 -22
- flwr/proto/exec_pb2_grpc.py +34 -0
- flwr/proto/exec_pb2_grpc.pyi +13 -0
- flwr/proto/log_pb2.py +29 -0
- flwr/proto/log_pb2.pyi +39 -0
- flwr/proto/log_pb2_grpc.py +4 -0
- flwr/proto/log_pb2_grpc.pyi +4 -0
- flwr/proto/message_pb2.py +8 -8
- flwr/proto/message_pb2.pyi +4 -1
- flwr/proto/run_pb2.py +32 -27
- flwr/proto/run_pb2.pyi +44 -1
- flwr/proto/serverappio_pb2.py +52 -0
- flwr/proto/{driver_pb2.pyi → serverappio_pb2.pyi} +54 -0
- flwr/proto/serverappio_pb2_grpc.py +376 -0
- flwr/proto/serverappio_pb2_grpc.pyi +147 -0
- flwr/proto/simulationio_pb2.py +38 -0
- flwr/proto/simulationio_pb2.pyi +65 -0
- flwr/proto/simulationio_pb2_grpc.py +205 -0
- flwr/proto/simulationio_pb2_grpc.pyi +81 -0
- flwr/server/app.py +297 -162
- flwr/server/driver/driver.py +15 -1
- flwr/server/driver/grpc_driver.py +89 -50
- flwr/server/driver/inmemory_driver.py +6 -16
- flwr/server/run_serverapp.py +11 -235
- flwr/server/{superlink/state → serverapp}/__init__.py +3 -9
- flwr/server/serverapp/app.py +234 -0
- flwr/server/strategy/aggregate.py +4 -4
- flwr/server/strategy/fedadam.py +11 -1
- flwr/server/superlink/driver/__init__.py +1 -1
- flwr/server/superlink/driver/{driver_grpc.py → serverappio_grpc.py} +19 -16
- flwr/server/superlink/driver/{driver_servicer.py → serverappio_servicer.py} +125 -39
- flwr/server/superlink/fleet/grpc_adapter/grpc_adapter_servicer.py +4 -2
- flwr/server/superlink/fleet/grpc_bidi/grpc_server.py +2 -2
- flwr/server/superlink/fleet/grpc_rere/fleet_servicer.py +4 -2
- flwr/server/superlink/fleet/grpc_rere/server_interceptor.py +2 -2
- flwr/server/superlink/fleet/message_handler/message_handler.py +7 -7
- flwr/server/superlink/fleet/rest_rere/rest_api.py +10 -9
- flwr/server/superlink/fleet/vce/vce_api.py +23 -23
- flwr/server/superlink/linkstate/__init__.py +28 -0
- flwr/server/superlink/{state/in_memory_state.py → linkstate/in_memory_linkstate.py} +237 -64
- flwr/server/superlink/{state/state.py → linkstate/linkstate.py} +166 -22
- flwr/server/superlink/{state/state_factory.py → linkstate/linkstate_factory.py} +9 -9
- flwr/server/superlink/{state/sqlite_state.py → linkstate/sqlite_linkstate.py} +383 -174
- flwr/server/superlink/linkstate/utils.py +389 -0
- flwr/server/superlink/simulation/__init__.py +15 -0
- flwr/server/superlink/simulation/simulationio_grpc.py +65 -0
- flwr/server/superlink/simulation/simulationio_servicer.py +153 -0
- flwr/simulation/__init__.py +5 -1
- flwr/simulation/app.py +236 -347
- flwr/simulation/legacy_app.py +402 -0
- flwr/simulation/ray_transport/ray_client_proxy.py +2 -2
- flwr/simulation/run_simulation.py +56 -141
- flwr/simulation/simulationio_connection.py +86 -0
- flwr/superexec/app.py +6 -134
- flwr/superexec/deployment.py +70 -69
- flwr/superexec/exec_grpc.py +15 -8
- flwr/superexec/exec_servicer.py +65 -65
- flwr/superexec/executor.py +26 -7
- flwr/superexec/simulation.py +62 -150
- {flwr-1.12.0.dist-info → flwr-1.13.1.dist-info}/METADATA +9 -7
- {flwr-1.12.0.dist-info → flwr-1.13.1.dist-info}/RECORD +105 -85
- {flwr-1.12.0.dist-info → flwr-1.13.1.dist-info}/entry_points.txt +2 -0
- flwr/client/node_state_tests.py +0 -66
- flwr/proto/driver_pb2.py +0 -42
- flwr/proto/driver_pb2_grpc.py +0 -239
- flwr/proto/driver_pb2_grpc.pyi +0 -94
- flwr/server/superlink/state/utils.py +0 -148
- {flwr-1.12.0.dist-info → flwr-1.13.1.dist-info}/LICENSE +0 -0
- {flwr-1.12.0.dist-info → flwr-1.13.1.dist-info}/WHEEL +0 -0
flwr/server/app.py
CHANGED
|
@@ -17,12 +17,13 @@
|
|
|
17
17
|
import argparse
|
|
18
18
|
import csv
|
|
19
19
|
import importlib.util
|
|
20
|
+
import subprocess
|
|
20
21
|
import sys
|
|
21
22
|
import threading
|
|
22
23
|
from collections.abc import Sequence
|
|
23
|
-
from logging import INFO, WARN
|
|
24
|
-
from os.path import isfile
|
|
24
|
+
from logging import DEBUG, INFO, WARN
|
|
25
25
|
from pathlib import Path
|
|
26
|
+
from time import sleep
|
|
26
27
|
from typing import Optional
|
|
27
28
|
|
|
28
29
|
import grpc
|
|
@@ -35,19 +36,26 @@ from cryptography.hazmat.primitives.serialization import (
|
|
|
35
36
|
|
|
36
37
|
from flwr.common import GRPC_MAX_MESSAGE_LENGTH, EventType, event
|
|
37
38
|
from flwr.common.address import parse_address
|
|
38
|
-
from flwr.common.
|
|
39
|
+
from flwr.common.args import try_obtain_server_certificates
|
|
40
|
+
from flwr.common.config import get_flwr_dir, parse_config_args
|
|
39
41
|
from flwr.common.constant import (
|
|
40
|
-
|
|
42
|
+
CLIENT_OCTET,
|
|
43
|
+
EXEC_API_DEFAULT_SERVER_ADDRESS,
|
|
41
44
|
FLEET_API_GRPC_BIDI_DEFAULT_ADDRESS,
|
|
42
45
|
FLEET_API_GRPC_RERE_DEFAULT_ADDRESS,
|
|
43
46
|
FLEET_API_REST_DEFAULT_ADDRESS,
|
|
47
|
+
ISOLATION_MODE_PROCESS,
|
|
48
|
+
ISOLATION_MODE_SUBPROCESS,
|
|
44
49
|
MISSING_EXTRA_REST,
|
|
50
|
+
SERVER_OCTET,
|
|
51
|
+
SERVERAPPIO_API_DEFAULT_SERVER_ADDRESS,
|
|
52
|
+
SIMULATIONIO_API_DEFAULT_SERVER_ADDRESS,
|
|
45
53
|
TRANSPORT_TYPE_GRPC_ADAPTER,
|
|
46
54
|
TRANSPORT_TYPE_GRPC_RERE,
|
|
47
55
|
TRANSPORT_TYPE_REST,
|
|
48
56
|
)
|
|
49
57
|
from flwr.common.exit_handlers import register_exit_handlers
|
|
50
|
-
from flwr.common.logger import log
|
|
58
|
+
from flwr.common.logger import log, warn_deprecated_feature
|
|
51
59
|
from flwr.common.secure_aggregation.crypto.symmetric_encryption import (
|
|
52
60
|
private_key_to_bytes,
|
|
53
61
|
public_key_to_bytes,
|
|
@@ -56,13 +64,15 @@ from flwr.proto.fleet_pb2_grpc import ( # pylint: disable=E0611
|
|
|
56
64
|
add_FleetServicer_to_server,
|
|
57
65
|
)
|
|
58
66
|
from flwr.proto.grpcadapter_pb2_grpc import add_GrpcAdapterServicer_to_server
|
|
67
|
+
from flwr.superexec.app import load_executor
|
|
68
|
+
from flwr.superexec.exec_grpc import run_exec_api_grpc
|
|
59
69
|
|
|
60
70
|
from .client_manager import ClientManager
|
|
61
71
|
from .history import History
|
|
62
72
|
from .server import Server, init_defaults, run_fl
|
|
63
73
|
from .server_config import ServerConfig
|
|
64
74
|
from .strategy import Strategy
|
|
65
|
-
from .superlink.driver.
|
|
75
|
+
from .superlink.driver.serverappio_grpc import run_serverappio_api_grpc
|
|
66
76
|
from .superlink.ffs.ffs_factory import FfsFactory
|
|
67
77
|
from .superlink.fleet.grpc_adapter.grpc_adapter_servicer import GrpcAdapterServicer
|
|
68
78
|
from .superlink.fleet.grpc_bidi.grpc_server import (
|
|
@@ -71,7 +81,8 @@ from .superlink.fleet.grpc_bidi.grpc_server import (
|
|
|
71
81
|
)
|
|
72
82
|
from .superlink.fleet.grpc_rere.fleet_servicer import FleetServicer
|
|
73
83
|
from .superlink.fleet.grpc_rere.server_interceptor import AuthenticateServerInterceptor
|
|
74
|
-
from .superlink.
|
|
84
|
+
from .superlink.linkstate import LinkStateFactory
|
|
85
|
+
from .superlink.simulation.simulationio_grpc import run_simulationio_api_grpc
|
|
75
86
|
|
|
76
87
|
DATABASE = ":flwr-in-memory-state:"
|
|
77
88
|
BASE_DIR = get_flwr_dir() / "superlink" / "ffs"
|
|
@@ -89,6 +100,11 @@ def start_server( # pylint: disable=too-many-arguments,too-many-locals
|
|
|
89
100
|
) -> History:
|
|
90
101
|
"""Start a Flower server using the gRPC transport layer.
|
|
91
102
|
|
|
103
|
+
Warning
|
|
104
|
+
-------
|
|
105
|
+
This function is deprecated since 1.13.0. Use the :code:`flower-superlink` command
|
|
106
|
+
instead to start a SuperLink.
|
|
107
|
+
|
|
92
108
|
Parameters
|
|
93
109
|
----------
|
|
94
110
|
server_address : Optional[str]
|
|
@@ -146,6 +162,17 @@ def start_server( # pylint: disable=too-many-arguments,too-many-locals
|
|
|
146
162
|
>>> )
|
|
147
163
|
>>> )
|
|
148
164
|
"""
|
|
165
|
+
msg = (
|
|
166
|
+
"flwr.server.start_server() is deprecated."
|
|
167
|
+
"\n\tInstead, use the `flower-superlink` CLI command to start a SuperLink "
|
|
168
|
+
"as shown below:"
|
|
169
|
+
"\n\n\t\t$ flower-superlink --insecure"
|
|
170
|
+
"\n\n\tTo view usage and all available options, run:"
|
|
171
|
+
"\n\n\t\t$ flower-superlink --help"
|
|
172
|
+
"\n\n\tUsing `start_server()` is deprecated."
|
|
173
|
+
)
|
|
174
|
+
warn_deprecated_feature(name=msg)
|
|
175
|
+
|
|
149
176
|
event(EventType.START_SERVER_ENTER)
|
|
150
177
|
|
|
151
178
|
# Parse IP address
|
|
@@ -198,125 +225,186 @@ def start_server( # pylint: disable=too-many-arguments,too-many-locals
|
|
|
198
225
|
|
|
199
226
|
# pylint: disable=too-many-branches, too-many-locals, too-many-statements
|
|
200
227
|
def run_superlink() -> None:
|
|
201
|
-
"""Run Flower SuperLink (
|
|
228
|
+
"""Run Flower SuperLink (ServerAppIo API and Fleet API)."""
|
|
202
229
|
args = _parse_args_run_superlink().parse_args()
|
|
203
230
|
|
|
204
231
|
log(INFO, "Starting Flower SuperLink")
|
|
205
232
|
|
|
206
233
|
event(EventType.RUN_SUPERLINK_ENTER)
|
|
207
234
|
|
|
208
|
-
#
|
|
209
|
-
|
|
235
|
+
# Warn unused options
|
|
236
|
+
if args.flwr_dir is not None:
|
|
237
|
+
log(
|
|
238
|
+
WARN, "The `--flwr-dir` option is currently not in use and will be ignored."
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
# Parse IP addresses
|
|
242
|
+
serverappio_address, _, _ = _format_address(args.serverappio_api_address)
|
|
243
|
+
exec_address, _, _ = _format_address(args.exec_api_address)
|
|
244
|
+
simulationio_address, _, _ = _format_address(args.simulationio_api_address)
|
|
210
245
|
|
|
211
246
|
# Obtain certificates
|
|
212
|
-
certificates =
|
|
247
|
+
certificates = try_obtain_server_certificates(args, args.fleet_api_type)
|
|
213
248
|
|
|
214
249
|
# Initialize StateFactory
|
|
215
|
-
state_factory =
|
|
250
|
+
state_factory = LinkStateFactory(args.database)
|
|
216
251
|
|
|
217
252
|
# Initialize FfsFactory
|
|
218
253
|
ffs_factory = FfsFactory(args.storage_dir)
|
|
219
254
|
|
|
220
|
-
# Start
|
|
221
|
-
|
|
222
|
-
|
|
255
|
+
# Start Exec API
|
|
256
|
+
executor = load_executor(args)
|
|
257
|
+
exec_server: grpc.Server = run_exec_api_grpc(
|
|
258
|
+
address=exec_address,
|
|
223
259
|
state_factory=state_factory,
|
|
224
260
|
ffs_factory=ffs_factory,
|
|
261
|
+
executor=executor,
|
|
225
262
|
certificates=certificates,
|
|
263
|
+
config=parse_config_args(
|
|
264
|
+
[args.executor_config] if args.executor_config else args.executor_config
|
|
265
|
+
),
|
|
226
266
|
)
|
|
267
|
+
grpc_servers = [exec_server]
|
|
227
268
|
|
|
228
|
-
|
|
269
|
+
# Determine Exec plugin
|
|
270
|
+
# If simulation is used, don't start ServerAppIo and Fleet APIs
|
|
271
|
+
sim_exec = executor.__class__.__qualname__ == "SimulationEngine"
|
|
229
272
|
bckg_threads = []
|
|
230
|
-
if not args.fleet_api_address:
|
|
231
|
-
if args.fleet_api_type in [
|
|
232
|
-
TRANSPORT_TYPE_GRPC_RERE,
|
|
233
|
-
TRANSPORT_TYPE_GRPC_ADAPTER,
|
|
234
|
-
]:
|
|
235
|
-
args.fleet_api_address = FLEET_API_GRPC_RERE_DEFAULT_ADDRESS
|
|
236
|
-
elif args.fleet_api_type == TRANSPORT_TYPE_REST:
|
|
237
|
-
args.fleet_api_address = FLEET_API_REST_DEFAULT_ADDRESS
|
|
238
|
-
|
|
239
|
-
fleet_address, host, port = _format_address(args.fleet_api_address)
|
|
240
273
|
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
"Support for multiple workers will be added in future releases. "
|
|
248
|
-
"Proceeding with a single worker.",
|
|
249
|
-
args.fleet_api_num_workers,
|
|
250
|
-
)
|
|
251
|
-
num_workers = 1
|
|
252
|
-
|
|
253
|
-
# Start Fleet API
|
|
254
|
-
if args.fleet_api_type == TRANSPORT_TYPE_REST:
|
|
255
|
-
if (
|
|
256
|
-
importlib.util.find_spec("requests")
|
|
257
|
-
and importlib.util.find_spec("starlette")
|
|
258
|
-
and importlib.util.find_spec("uvicorn")
|
|
259
|
-
) is None:
|
|
260
|
-
sys.exit(MISSING_EXTRA_REST)
|
|
261
|
-
|
|
262
|
-
_, ssl_certfile, ssl_keyfile = (
|
|
263
|
-
certificates if certificates is not None else (None, None, None)
|
|
274
|
+
if sim_exec:
|
|
275
|
+
simulationio_server: grpc.Server = run_simulationio_api_grpc(
|
|
276
|
+
address=simulationio_address,
|
|
277
|
+
state_factory=state_factory,
|
|
278
|
+
ffs_factory=ffs_factory,
|
|
279
|
+
certificates=None, # SimulationAppIo API doesn't support SSL yet
|
|
264
280
|
)
|
|
281
|
+
grpc_servers.append(simulationio_server)
|
|
265
282
|
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
state_factory,
|
|
274
|
-
ffs_factory,
|
|
275
|
-
num_workers,
|
|
276
|
-
),
|
|
283
|
+
else:
|
|
284
|
+
# Start ServerAppIo API
|
|
285
|
+
serverappio_server: grpc.Server = run_serverappio_api_grpc(
|
|
286
|
+
address=serverappio_address,
|
|
287
|
+
state_factory=state_factory,
|
|
288
|
+
ffs_factory=ffs_factory,
|
|
289
|
+
certificates=None, # ServerAppIo API doesn't support SSL yet
|
|
277
290
|
)
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
)
|
|
291
|
+
grpc_servers.append(serverappio_server)
|
|
292
|
+
|
|
293
|
+
# Start Fleet API
|
|
294
|
+
if not args.fleet_api_address:
|
|
295
|
+
if args.fleet_api_type in [
|
|
296
|
+
TRANSPORT_TYPE_GRPC_RERE,
|
|
297
|
+
TRANSPORT_TYPE_GRPC_ADAPTER,
|
|
298
|
+
]:
|
|
299
|
+
args.fleet_api_address = FLEET_API_GRPC_RERE_DEFAULT_ADDRESS
|
|
300
|
+
elif args.fleet_api_type == TRANSPORT_TYPE_REST:
|
|
301
|
+
args.fleet_api_address = FLEET_API_REST_DEFAULT_ADDRESS
|
|
302
|
+
|
|
303
|
+
fleet_address, host, port = _format_address(args.fleet_api_address)
|
|
304
|
+
|
|
305
|
+
num_workers = args.fleet_api_num_workers
|
|
306
|
+
if num_workers != 1:
|
|
295
307
|
log(
|
|
296
|
-
|
|
297
|
-
"
|
|
298
|
-
|
|
308
|
+
WARN,
|
|
309
|
+
"The Fleet API currently supports only 1 worker. "
|
|
310
|
+
"You have specified %d workers. "
|
|
311
|
+
"Support for multiple workers will be added in future releases. "
|
|
312
|
+
"Proceeding with a single worker.",
|
|
313
|
+
args.fleet_api_num_workers,
|
|
314
|
+
)
|
|
315
|
+
num_workers = 1
|
|
316
|
+
|
|
317
|
+
if args.fleet_api_type == TRANSPORT_TYPE_REST:
|
|
318
|
+
if (
|
|
319
|
+
importlib.util.find_spec("requests")
|
|
320
|
+
and importlib.util.find_spec("starlette")
|
|
321
|
+
and importlib.util.find_spec("uvicorn")
|
|
322
|
+
) is None:
|
|
323
|
+
sys.exit(MISSING_EXTRA_REST)
|
|
324
|
+
|
|
325
|
+
_, ssl_certfile, ssl_keyfile = (
|
|
326
|
+
certificates if certificates is not None else (None, None, None)
|
|
299
327
|
)
|
|
300
|
-
interceptors = [AuthenticateServerInterceptor(state)]
|
|
301
328
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
329
|
+
fleet_thread = threading.Thread(
|
|
330
|
+
target=_run_fleet_api_rest,
|
|
331
|
+
args=(
|
|
332
|
+
host,
|
|
333
|
+
port,
|
|
334
|
+
ssl_keyfile,
|
|
335
|
+
ssl_certfile,
|
|
336
|
+
state_factory,
|
|
337
|
+
ffs_factory,
|
|
338
|
+
num_workers,
|
|
339
|
+
),
|
|
340
|
+
)
|
|
341
|
+
fleet_thread.start()
|
|
342
|
+
bckg_threads.append(fleet_thread)
|
|
343
|
+
elif args.fleet_api_type == TRANSPORT_TYPE_GRPC_RERE:
|
|
344
|
+
maybe_keys = _try_setup_node_authentication(args, certificates)
|
|
345
|
+
interceptors: Optional[Sequence[grpc.ServerInterceptor]] = None
|
|
346
|
+
if maybe_keys is not None:
|
|
347
|
+
(
|
|
348
|
+
node_public_keys,
|
|
349
|
+
server_private_key,
|
|
350
|
+
server_public_key,
|
|
351
|
+
) = maybe_keys
|
|
352
|
+
state = state_factory.state()
|
|
353
|
+
state.store_node_public_keys(node_public_keys)
|
|
354
|
+
state.store_server_private_public_key(
|
|
355
|
+
private_key_to_bytes(server_private_key),
|
|
356
|
+
public_key_to_bytes(server_public_key),
|
|
357
|
+
)
|
|
358
|
+
log(
|
|
359
|
+
INFO,
|
|
360
|
+
"Node authentication enabled with %d known public keys",
|
|
361
|
+
len(node_public_keys),
|
|
362
|
+
)
|
|
363
|
+
interceptors = [AuthenticateServerInterceptor(state)]
|
|
364
|
+
|
|
365
|
+
fleet_server = _run_fleet_api_grpc_rere(
|
|
366
|
+
address=fleet_address,
|
|
367
|
+
state_factory=state_factory,
|
|
368
|
+
ffs_factory=ffs_factory,
|
|
369
|
+
certificates=certificates,
|
|
370
|
+
interceptors=interceptors,
|
|
371
|
+
)
|
|
372
|
+
grpc_servers.append(fleet_server)
|
|
373
|
+
elif args.fleet_api_type == TRANSPORT_TYPE_GRPC_ADAPTER:
|
|
374
|
+
fleet_server = _run_fleet_api_grpc_adapter(
|
|
375
|
+
address=fleet_address,
|
|
376
|
+
state_factory=state_factory,
|
|
377
|
+
ffs_factory=ffs_factory,
|
|
378
|
+
certificates=certificates,
|
|
379
|
+
)
|
|
380
|
+
grpc_servers.append(fleet_server)
|
|
381
|
+
else:
|
|
382
|
+
raise ValueError(f"Unknown fleet_api_type: {args.fleet_api_type}")
|
|
383
|
+
|
|
384
|
+
if args.isolation == ISOLATION_MODE_SUBPROCESS:
|
|
385
|
+
|
|
386
|
+
_octet, _colon, _port = serverappio_address.rpartition(":")
|
|
387
|
+
io_address = (
|
|
388
|
+
f"{CLIENT_OCTET}:{_port}" if _octet == SERVER_OCTET else serverappio_address
|
|
308
389
|
)
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
fleet_server = _run_fleet_api_grpc_adapter(
|
|
312
|
-
address=fleet_address,
|
|
313
|
-
state_factory=state_factory,
|
|
314
|
-
ffs_factory=ffs_factory,
|
|
315
|
-
certificates=certificates,
|
|
390
|
+
address_arg = (
|
|
391
|
+
"--simulationio-api-address" if sim_exec else "--serverappio-api-address"
|
|
316
392
|
)
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
393
|
+
address = simulationio_address if sim_exec else io_address
|
|
394
|
+
cmd = "flwr-simulation" if sim_exec else "flwr-serverapp"
|
|
395
|
+
|
|
396
|
+
# Scheduler thread
|
|
397
|
+
scheduler_th = threading.Thread(
|
|
398
|
+
target=_flwr_scheduler,
|
|
399
|
+
args=(
|
|
400
|
+
state_factory,
|
|
401
|
+
address_arg,
|
|
402
|
+
address,
|
|
403
|
+
cmd,
|
|
404
|
+
),
|
|
405
|
+
)
|
|
406
|
+
scheduler_th.start()
|
|
407
|
+
bckg_threads.append(scheduler_th)
|
|
320
408
|
|
|
321
409
|
# Graceful shutdown
|
|
322
410
|
register_exit_handlers(
|
|
@@ -331,7 +419,45 @@ def run_superlink() -> None:
|
|
|
331
419
|
for thread in bckg_threads:
|
|
332
420
|
if not thread.is_alive():
|
|
333
421
|
sys.exit(1)
|
|
334
|
-
|
|
422
|
+
exec_server.wait_for_termination(timeout=1)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _flwr_scheduler(
|
|
426
|
+
state_factory: LinkStateFactory,
|
|
427
|
+
io_api_arg: str,
|
|
428
|
+
io_api_address: str,
|
|
429
|
+
cmd: str,
|
|
430
|
+
) -> None:
|
|
431
|
+
log(DEBUG, "Started %s scheduler thread.", cmd)
|
|
432
|
+
|
|
433
|
+
state = state_factory.state()
|
|
434
|
+
|
|
435
|
+
# Periodically check for a pending run in the LinkState
|
|
436
|
+
while True:
|
|
437
|
+
sleep(3)
|
|
438
|
+
pending_run_id = state.get_pending_run_id()
|
|
439
|
+
|
|
440
|
+
if pending_run_id:
|
|
441
|
+
|
|
442
|
+
log(
|
|
443
|
+
INFO,
|
|
444
|
+
"Launching %s subprocess. Connects to SuperLink on %s",
|
|
445
|
+
cmd,
|
|
446
|
+
io_api_address,
|
|
447
|
+
)
|
|
448
|
+
# Start subprocess
|
|
449
|
+
command = [
|
|
450
|
+
cmd,
|
|
451
|
+
"--run-once",
|
|
452
|
+
io_api_arg,
|
|
453
|
+
io_api_address,
|
|
454
|
+
"--insecure",
|
|
455
|
+
]
|
|
456
|
+
|
|
457
|
+
subprocess.Popen( # pylint: disable=consider-using-with
|
|
458
|
+
command,
|
|
459
|
+
text=True,
|
|
460
|
+
)
|
|
335
461
|
|
|
336
462
|
|
|
337
463
|
def _format_address(address: str) -> tuple[str, str, int]:
|
|
@@ -433,63 +559,9 @@ def _try_setup_node_authentication(
|
|
|
433
559
|
)
|
|
434
560
|
|
|
435
561
|
|
|
436
|
-
def _try_obtain_certificates(
|
|
437
|
-
args: argparse.Namespace,
|
|
438
|
-
) -> Optional[tuple[bytes, bytes, bytes]]:
|
|
439
|
-
# Obtain certificates
|
|
440
|
-
if args.insecure:
|
|
441
|
-
log(WARN, "Option `--insecure` was set. Starting insecure HTTP server.")
|
|
442
|
-
return None
|
|
443
|
-
# Check if certificates are provided
|
|
444
|
-
if args.fleet_api_type in [TRANSPORT_TYPE_GRPC_RERE, TRANSPORT_TYPE_GRPC_ADAPTER]:
|
|
445
|
-
if args.ssl_certfile and args.ssl_keyfile and args.ssl_ca_certfile:
|
|
446
|
-
if not isfile(args.ssl_ca_certfile):
|
|
447
|
-
sys.exit("Path argument `--ssl-ca-certfile` does not point to a file.")
|
|
448
|
-
if not isfile(args.ssl_certfile):
|
|
449
|
-
sys.exit("Path argument `--ssl-certfile` does not point to a file.")
|
|
450
|
-
if not isfile(args.ssl_keyfile):
|
|
451
|
-
sys.exit("Path argument `--ssl-keyfile` does not point to a file.")
|
|
452
|
-
certificates = (
|
|
453
|
-
Path(args.ssl_ca_certfile).read_bytes(), # CA certificate
|
|
454
|
-
Path(args.ssl_certfile).read_bytes(), # server certificate
|
|
455
|
-
Path(args.ssl_keyfile).read_bytes(), # server private key
|
|
456
|
-
)
|
|
457
|
-
return certificates
|
|
458
|
-
if args.ssl_certfile or args.ssl_keyfile or args.ssl_ca_certfile:
|
|
459
|
-
sys.exit(
|
|
460
|
-
"You need to provide valid file paths to `--ssl-certfile`, "
|
|
461
|
-
"`--ssl-keyfile`, and `—-ssl-ca-certfile` to create a secure "
|
|
462
|
-
"connection in Fleet API server (gRPC-rere)."
|
|
463
|
-
)
|
|
464
|
-
if args.fleet_api_type == TRANSPORT_TYPE_REST:
|
|
465
|
-
if args.ssl_certfile and args.ssl_keyfile:
|
|
466
|
-
if not isfile(args.ssl_certfile):
|
|
467
|
-
sys.exit("Path argument `--ssl-certfile` does not point to a file.")
|
|
468
|
-
if not isfile(args.ssl_keyfile):
|
|
469
|
-
sys.exit("Path argument `--ssl-keyfile` does not point to a file.")
|
|
470
|
-
certificates = (
|
|
471
|
-
b"",
|
|
472
|
-
Path(args.ssl_certfile).read_bytes(), # server certificate
|
|
473
|
-
Path(args.ssl_keyfile).read_bytes(), # server private key
|
|
474
|
-
)
|
|
475
|
-
return certificates
|
|
476
|
-
if args.ssl_certfile or args.ssl_keyfile:
|
|
477
|
-
sys.exit(
|
|
478
|
-
"You need to provide valid file paths to `--ssl-certfile` "
|
|
479
|
-
"and `--ssl-keyfile` to create a secure connection "
|
|
480
|
-
"in Fleet API server (REST, experimental)."
|
|
481
|
-
)
|
|
482
|
-
sys.exit(
|
|
483
|
-
"Certificates are required unless running in insecure mode. "
|
|
484
|
-
"Please provide certificate paths to `--ssl-certfile`, "
|
|
485
|
-
"`--ssl-keyfile`, and `—-ssl-ca-certfile` or run the server "
|
|
486
|
-
"in insecure mode using '--insecure' if you understand the risks."
|
|
487
|
-
)
|
|
488
|
-
|
|
489
|
-
|
|
490
562
|
def _run_fleet_api_grpc_rere(
|
|
491
563
|
address: str,
|
|
492
|
-
state_factory:
|
|
564
|
+
state_factory: LinkStateFactory,
|
|
493
565
|
ffs_factory: FfsFactory,
|
|
494
566
|
certificates: Optional[tuple[bytes, bytes, bytes]],
|
|
495
567
|
interceptors: Optional[Sequence[grpc.ServerInterceptor]] = None,
|
|
@@ -517,7 +589,7 @@ def _run_fleet_api_grpc_rere(
|
|
|
517
589
|
|
|
518
590
|
def _run_fleet_api_grpc_adapter(
|
|
519
591
|
address: str,
|
|
520
|
-
state_factory:
|
|
592
|
+
state_factory: LinkStateFactory,
|
|
521
593
|
ffs_factory: FfsFactory,
|
|
522
594
|
certificates: Optional[tuple[bytes, bytes, bytes]],
|
|
523
595
|
) -> grpc.Server:
|
|
@@ -548,11 +620,11 @@ def _run_fleet_api_rest(
|
|
|
548
620
|
port: int,
|
|
549
621
|
ssl_keyfile: Optional[str],
|
|
550
622
|
ssl_certfile: Optional[str],
|
|
551
|
-
state_factory:
|
|
623
|
+
state_factory: LinkStateFactory,
|
|
552
624
|
ffs_factory: FfsFactory,
|
|
553
625
|
num_workers: int,
|
|
554
626
|
) -> None:
|
|
555
|
-
"""Run
|
|
627
|
+
"""Run ServerAppIo API (REST-based)."""
|
|
556
628
|
try:
|
|
557
629
|
import uvicorn
|
|
558
630
|
|
|
@@ -579,14 +651,16 @@ def _run_fleet_api_rest(
|
|
|
579
651
|
|
|
580
652
|
|
|
581
653
|
def _parse_args_run_superlink() -> argparse.ArgumentParser:
|
|
582
|
-
"""Parse command line arguments for both
|
|
654
|
+
"""Parse command line arguments for both ServerAppIo API and Fleet API."""
|
|
583
655
|
parser = argparse.ArgumentParser(
|
|
584
656
|
description="Start a Flower SuperLink",
|
|
585
657
|
)
|
|
586
658
|
|
|
587
659
|
_add_args_common(parser=parser)
|
|
588
|
-
|
|
660
|
+
_add_args_serverappio_api(parser=parser)
|
|
589
661
|
_add_args_fleet_api(parser=parser)
|
|
662
|
+
_add_args_exec_api(parser=parser)
|
|
663
|
+
_add_args_simulationio_api(parser=parser)
|
|
590
664
|
|
|
591
665
|
return parser
|
|
592
666
|
|
|
@@ -599,6 +673,17 @@ def _add_args_common(parser: argparse.ArgumentParser) -> None:
|
|
|
599
673
|
"paths are provided. By default, the server runs with HTTPS enabled. "
|
|
600
674
|
"Use this flag only if you understand the risks.",
|
|
601
675
|
)
|
|
676
|
+
parser.add_argument(
|
|
677
|
+
"--flwr-dir",
|
|
678
|
+
default=None,
|
|
679
|
+
help="""The path containing installed Flower Apps.
|
|
680
|
+
The default directory is:
|
|
681
|
+
|
|
682
|
+
- `$FLWR_HOME/` if `$FLWR_HOME` is defined
|
|
683
|
+
- `$XDG_DATA_HOME/.flwr/` if `$XDG_DATA_HOME` is defined
|
|
684
|
+
- `$HOME/.flwr/` in all other cases
|
|
685
|
+
""",
|
|
686
|
+
)
|
|
602
687
|
parser.add_argument(
|
|
603
688
|
"--ssl-certfile",
|
|
604
689
|
help="Fleet API server SSL certificate file (as a path str) "
|
|
@@ -618,6 +703,19 @@ def _add_args_common(parser: argparse.ArgumentParser) -> None:
|
|
|
618
703
|
"to create a secure connection.",
|
|
619
704
|
type=str,
|
|
620
705
|
)
|
|
706
|
+
parser.add_argument(
|
|
707
|
+
"--isolation",
|
|
708
|
+
default=ISOLATION_MODE_SUBPROCESS,
|
|
709
|
+
required=False,
|
|
710
|
+
choices=[
|
|
711
|
+
ISOLATION_MODE_SUBPROCESS,
|
|
712
|
+
ISOLATION_MODE_PROCESS,
|
|
713
|
+
],
|
|
714
|
+
help="Isolation mode when running a `ServerApp` (`subprocess` by default, "
|
|
715
|
+
"possible values: `subprocess`, `process`). Use `subprocess` to configure "
|
|
716
|
+
"SuperLink to run a `ServerApp` in a subprocess. Use `process` to indicate "
|
|
717
|
+
"that a separate independent process gets created outside of SuperLink.",
|
|
718
|
+
)
|
|
621
719
|
parser.add_argument(
|
|
622
720
|
"--database",
|
|
623
721
|
help="A string representing the path to the database "
|
|
@@ -650,11 +748,12 @@ def _add_args_common(parser: argparse.ArgumentParser) -> None:
|
|
|
650
748
|
)
|
|
651
749
|
|
|
652
750
|
|
|
653
|
-
def
|
|
751
|
+
def _add_args_serverappio_api(parser: argparse.ArgumentParser) -> None:
|
|
654
752
|
parser.add_argument(
|
|
655
|
-
"--
|
|
656
|
-
|
|
657
|
-
|
|
753
|
+
"--serverappio-api-address",
|
|
754
|
+
default=SERVERAPPIO_API_DEFAULT_SERVER_ADDRESS,
|
|
755
|
+
help="ServerAppIo API (gRPC) server address (IPv4, IPv6, or a domain name). "
|
|
756
|
+
f"By default, it is set to {SERVERAPPIO_API_DEFAULT_SERVER_ADDRESS}.",
|
|
658
757
|
)
|
|
659
758
|
|
|
660
759
|
|
|
@@ -681,3 +780,39 @@ def _add_args_fleet_api(parser: argparse.ArgumentParser) -> None:
|
|
|
681
780
|
type=int,
|
|
682
781
|
help="Set the number of concurrent workers for the Fleet API server.",
|
|
683
782
|
)
|
|
783
|
+
|
|
784
|
+
|
|
785
|
+
def _add_args_exec_api(parser: argparse.ArgumentParser) -> None:
|
|
786
|
+
"""Add command line arguments for Exec API."""
|
|
787
|
+
parser.add_argument(
|
|
788
|
+
"--exec-api-address",
|
|
789
|
+
help="Exec API server address (IPv4, IPv6, or a domain name) "
|
|
790
|
+
f"By default, it is set to {EXEC_API_DEFAULT_SERVER_ADDRESS}.",
|
|
791
|
+
default=EXEC_API_DEFAULT_SERVER_ADDRESS,
|
|
792
|
+
)
|
|
793
|
+
parser.add_argument(
|
|
794
|
+
"--executor",
|
|
795
|
+
help="For example: `deployment:exec` or `project.package.module:wrapper.exec`. "
|
|
796
|
+
"The default is `flwr.superexec.deployment:executor`",
|
|
797
|
+
default="flwr.superexec.deployment:executor",
|
|
798
|
+
)
|
|
799
|
+
parser.add_argument(
|
|
800
|
+
"--executor-dir",
|
|
801
|
+
help="The directory for the executor.",
|
|
802
|
+
default=".",
|
|
803
|
+
)
|
|
804
|
+
parser.add_argument(
|
|
805
|
+
"--executor-config",
|
|
806
|
+
help="Key-value pairs for the executor config, separated by spaces. "
|
|
807
|
+
"For example:\n\n`--executor-config 'verbose=true "
|
|
808
|
+
'root-certificates="certificates/superlink-ca.crt"\'`',
|
|
809
|
+
)
|
|
810
|
+
|
|
811
|
+
|
|
812
|
+
def _add_args_simulationio_api(parser: argparse.ArgumentParser) -> None:
|
|
813
|
+
parser.add_argument(
|
|
814
|
+
"--simulationio-api-address",
|
|
815
|
+
default=SIMULATIONIO_API_DEFAULT_SERVER_ADDRESS,
|
|
816
|
+
help="SimulationIo API (gRPC) server address (IPv4, IPv6, or a domain name)."
|
|
817
|
+
f"By default, it is set to {SIMULATIONIO_API_DEFAULT_SERVER_ADDRESS}.",
|
|
818
|
+
)
|
flwr/server/driver/driver.py
CHANGED
|
@@ -24,7 +24,21 @@ from flwr.common.typing import Run
|
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
class Driver(ABC):
|
|
27
|
-
"""Abstract base Driver class for the
|
|
27
|
+
"""Abstract base Driver class for the ServerAppIo API."""
|
|
28
|
+
|
|
29
|
+
@abstractmethod
|
|
30
|
+
def set_run(self, run_id: int) -> None:
|
|
31
|
+
"""Request a run to the SuperLink with a given `run_id`.
|
|
32
|
+
|
|
33
|
+
If a Run with the specified `run_id` exists, a local Run
|
|
34
|
+
object will be created. It enables further functionality
|
|
35
|
+
in the driver, such as sending `Messages`.
|
|
36
|
+
|
|
37
|
+
Parameters
|
|
38
|
+
----------
|
|
39
|
+
run_id : int
|
|
40
|
+
The `run_id` of the Run this Driver object operates in.
|
|
41
|
+
"""
|
|
28
42
|
|
|
29
43
|
@property
|
|
30
44
|
@abstractmethod
|