flwr-nightly 1.13.0.dev20241111__py3-none-any.whl → 1.14.0.dev20241126__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flwr-nightly might be problematic. Click here for more details.
- flwr/cli/app.py +2 -0
- flwr/cli/install.py +0 -16
- flwr/cli/ls.py +228 -0
- flwr/cli/new/new.py +23 -13
- flwr/cli/new/templates/app/README.md.tpl +11 -0
- flwr/cli/new/templates/app/pyproject.baseline.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.huggingface.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.jax.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.mlx.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.numpy.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.sklearn.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl +1 -1
- flwr/cli/run/run.py +4 -2
- flwr/client/app.py +50 -14
- flwr/client/clientapp/app.py +40 -23
- flwr/client/grpc_rere_client/connection.py +7 -12
- flwr/client/rest_client/connection.py +4 -14
- flwr/client/supernode/app.py +31 -53
- flwr/common/args.py +85 -16
- flwr/common/constant.py +24 -6
- flwr/common/date.py +18 -0
- flwr/common/grpc.py +4 -1
- flwr/common/serde.py +10 -0
- flwr/common/typing.py +31 -10
- flwr/proto/exec_pb2.py +22 -13
- flwr/proto/exec_pb2.pyi +44 -0
- flwr/proto/exec_pb2_grpc.py +34 -0
- flwr/proto/exec_pb2_grpc.pyi +13 -0
- flwr/proto/run_pb2.py +30 -30
- flwr/proto/run_pb2.pyi +18 -1
- flwr/server/app.py +47 -77
- flwr/server/driver/grpc_driver.py +66 -16
- flwr/server/run_serverapp.py +8 -238
- flwr/server/serverapp/app.py +49 -29
- flwr/server/superlink/fleet/rest_rere/rest_api.py +10 -9
- flwr/server/superlink/linkstate/in_memory_linkstate.py +71 -46
- flwr/server/superlink/linkstate/linkstate.py +19 -5
- flwr/server/superlink/linkstate/sqlite_linkstate.py +81 -113
- flwr/server/superlink/linkstate/utils.py +193 -3
- flwr/simulation/app.py +52 -91
- flwr/simulation/legacy_app.py +21 -1
- flwr/simulation/run_simulation.py +7 -18
- flwr/simulation/simulationio_connection.py +2 -2
- flwr/superexec/deployment.py +12 -6
- flwr/superexec/exec_servicer.py +31 -2
- flwr/superexec/simulation.py +11 -46
- {flwr_nightly-1.13.0.dev20241111.dist-info → flwr_nightly-1.14.0.dev20241126.dist-info}/METADATA +5 -4
- {flwr_nightly-1.13.0.dev20241111.dist-info → flwr_nightly-1.14.0.dev20241126.dist-info}/RECORD +53 -52
- {flwr_nightly-1.13.0.dev20241111.dist-info → flwr_nightly-1.14.0.dev20241126.dist-info}/LICENSE +0 -0
- {flwr_nightly-1.13.0.dev20241111.dist-info → flwr_nightly-1.14.0.dev20241126.dist-info}/WHEEL +0 -0
- {flwr_nightly-1.13.0.dev20241111.dist-info → flwr_nightly-1.14.0.dev20241126.dist-info}/entry_points.txt +0 -0
flwr/client/clientapp/app.py
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
"""Flower ClientApp process."""
|
|
16
16
|
|
|
17
17
|
import argparse
|
|
18
|
+
import sys
|
|
18
19
|
import time
|
|
19
20
|
from logging import DEBUG, ERROR, INFO
|
|
20
21
|
from typing import Optional
|
|
@@ -26,7 +27,7 @@ from flwr.client.client_app import ClientApp, LoadClientAppError
|
|
|
26
27
|
from flwr.common import Context, Message
|
|
27
28
|
from flwr.common.args import add_args_flwr_app_common
|
|
28
29
|
from flwr.common.config import get_flwr_dir
|
|
29
|
-
from flwr.common.constant import ErrorCode
|
|
30
|
+
from flwr.common.constant import CLIENTAPPIO_API_DEFAULT_CLIENT_ADDRESS, ErrorCode
|
|
30
31
|
from flwr.common.grpc import create_channel
|
|
31
32
|
from flwr.common.logger import log
|
|
32
33
|
from flwr.common.message import Error
|
|
@@ -56,37 +57,29 @@ from .utils import get_load_client_app_fn
|
|
|
56
57
|
|
|
57
58
|
def flwr_clientapp() -> None:
|
|
58
59
|
"""Run process-isolated Flower ClientApp."""
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
parser.add_argument(
|
|
68
|
-
"--token",
|
|
69
|
-
type=int,
|
|
70
|
-
required=False,
|
|
71
|
-
help="Unique token generated by SuperNode for each ClientApp execution",
|
|
72
|
-
)
|
|
73
|
-
add_args_flwr_app_common(parser=parser)
|
|
74
|
-
args = parser.parse_args()
|
|
60
|
+
args = _parse_args_run_flwr_clientapp().parse_args()
|
|
61
|
+
if not args.insecure:
|
|
62
|
+
log(
|
|
63
|
+
ERROR,
|
|
64
|
+
"flwr-clientapp does not support TLS yet. "
|
|
65
|
+
"Please use the '--insecure' flag.",
|
|
66
|
+
)
|
|
67
|
+
sys.exit(1)
|
|
75
68
|
|
|
76
69
|
log(INFO, "Starting Flower ClientApp")
|
|
77
|
-
|
|
78
70
|
log(
|
|
79
71
|
DEBUG,
|
|
80
72
|
"Starting isolated `ClientApp` connected to SuperNode's ClientAppIo API at %s "
|
|
81
73
|
"with token %s",
|
|
82
|
-
args.
|
|
74
|
+
args.clientappio_api_address,
|
|
83
75
|
args.token,
|
|
84
76
|
)
|
|
85
77
|
run_clientapp(
|
|
86
|
-
|
|
78
|
+
clientappio_api_address=args.clientappio_api_address,
|
|
87
79
|
run_once=(args.token is not None),
|
|
88
80
|
token=args.token,
|
|
89
81
|
flwr_dir=args.flwr_dir,
|
|
82
|
+
certificates=None,
|
|
90
83
|
)
|
|
91
84
|
|
|
92
85
|
|
|
@@ -96,15 +89,17 @@ def on_channel_state_change(channel_connectivity: str) -> None:
|
|
|
96
89
|
|
|
97
90
|
|
|
98
91
|
def run_clientapp( # pylint: disable=R0914
|
|
99
|
-
|
|
92
|
+
clientappio_api_address: str,
|
|
100
93
|
run_once: bool,
|
|
101
94
|
token: Optional[int] = None,
|
|
102
95
|
flwr_dir: Optional[str] = None,
|
|
96
|
+
certificates: Optional[bytes] = None,
|
|
103
97
|
) -> None:
|
|
104
98
|
"""Run Flower ClientApp process."""
|
|
105
99
|
channel = create_channel(
|
|
106
|
-
server_address=
|
|
107
|
-
insecure=
|
|
100
|
+
server_address=clientappio_api_address,
|
|
101
|
+
insecure=(certificates is None),
|
|
102
|
+
root_certificates=certificates,
|
|
108
103
|
)
|
|
109
104
|
channel.subscribe(on_channel_state_change)
|
|
110
105
|
|
|
@@ -237,3 +232,25 @@ def push_message(
|
|
|
237
232
|
except grpc.RpcError as e:
|
|
238
233
|
log(ERROR, "[PushClientAppOutputs] gRPC error occurred: %s", str(e))
|
|
239
234
|
raise e
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _parse_args_run_flwr_clientapp() -> argparse.ArgumentParser:
|
|
238
|
+
"""Parse flwr-clientapp command line arguments."""
|
|
239
|
+
parser = argparse.ArgumentParser(
|
|
240
|
+
description="Run a Flower ClientApp",
|
|
241
|
+
)
|
|
242
|
+
parser.add_argument(
|
|
243
|
+
"--clientappio-api-address",
|
|
244
|
+
default=CLIENTAPPIO_API_DEFAULT_CLIENT_ADDRESS,
|
|
245
|
+
type=str,
|
|
246
|
+
help="Address of SuperNode's ClientAppIo API (IPv4, IPv6, or a domain name)."
|
|
247
|
+
f"By default, it is set to {CLIENTAPPIO_API_DEFAULT_CLIENT_ADDRESS}.",
|
|
248
|
+
)
|
|
249
|
+
parser.add_argument(
|
|
250
|
+
"--token",
|
|
251
|
+
type=int,
|
|
252
|
+
required=False,
|
|
253
|
+
help="Unique token generated by SuperNode for each ClientApp execution",
|
|
254
|
+
)
|
|
255
|
+
add_args_flwr_app_common(parser=parser)
|
|
256
|
+
return parser
|
|
@@ -41,11 +41,7 @@ from flwr.common.grpc import create_channel
|
|
|
41
41
|
from flwr.common.logger import log
|
|
42
42
|
from flwr.common.message import Message, Metadata
|
|
43
43
|
from flwr.common.retry_invoker import RetryInvoker
|
|
44
|
-
from flwr.common.serde import
|
|
45
|
-
message_from_taskins,
|
|
46
|
-
message_to_taskres,
|
|
47
|
-
user_config_from_proto,
|
|
48
|
-
)
|
|
44
|
+
from flwr.common.serde import message_from_taskins, message_to_taskres, run_from_proto
|
|
49
45
|
from flwr.common.typing import Fab, Run
|
|
50
46
|
from flwr.proto.fab_pb2 import GetFabRequest, GetFabResponse # pylint: disable=E0611
|
|
51
47
|
from flwr.proto.fleet_pb2 import ( # pylint: disable=E0611
|
|
@@ -159,6 +155,11 @@ def grpc_request_response( # pylint: disable=R0913,R0914,R0915,R0917
|
|
|
159
155
|
ping_thread: Optional[threading.Thread] = None
|
|
160
156
|
ping_stop_event = threading.Event()
|
|
161
157
|
|
|
158
|
+
# Restrict retries to cases where the status code is UNAVAILABLE
|
|
159
|
+
retry_invoker.should_giveup = (
|
|
160
|
+
lambda e: e.code() != grpc.StatusCode.UNAVAILABLE # type: ignore
|
|
161
|
+
)
|
|
162
|
+
|
|
162
163
|
###########################################################################
|
|
163
164
|
# ping/create_node/delete_node/receive/send/get_run functions
|
|
164
165
|
###########################################################################
|
|
@@ -287,13 +288,7 @@ def grpc_request_response( # pylint: disable=R0913,R0914,R0915,R0917
|
|
|
287
288
|
)
|
|
288
289
|
|
|
289
290
|
# Return fab_id and fab_version
|
|
290
|
-
return
|
|
291
|
-
run_id,
|
|
292
|
-
get_run_response.run.fab_id,
|
|
293
|
-
get_run_response.run.fab_version,
|
|
294
|
-
get_run_response.run.fab_hash,
|
|
295
|
-
user_config_from_proto(get_run_response.run.override_config),
|
|
296
|
-
)
|
|
291
|
+
return run_from_proto(get_run_response.run)
|
|
297
292
|
|
|
298
293
|
def get_fab(fab_hash: str) -> Fab:
|
|
299
294
|
# Call FleetAPI
|
|
@@ -41,11 +41,7 @@ from flwr.common.constant import (
|
|
|
41
41
|
from flwr.common.logger import log
|
|
42
42
|
from flwr.common.message import Message, Metadata
|
|
43
43
|
from flwr.common.retry_invoker import RetryInvoker
|
|
44
|
-
from flwr.common.serde import
|
|
45
|
-
message_from_taskins,
|
|
46
|
-
message_to_taskres,
|
|
47
|
-
user_config_from_proto,
|
|
48
|
-
)
|
|
44
|
+
from flwr.common.serde import message_from_taskins, message_to_taskres, run_from_proto
|
|
49
45
|
from flwr.common.typing import Fab, Run
|
|
50
46
|
from flwr.proto.fab_pb2 import GetFabRequest, GetFabResponse # pylint: disable=E0611
|
|
51
47
|
from flwr.proto.fleet_pb2 import ( # pylint: disable=E0611
|
|
@@ -361,15 +357,9 @@ def http_request_response( # pylint: disable=R0913,R0914,R0915,R0917
|
|
|
361
357
|
# Send the request
|
|
362
358
|
res = _request(req, GetRunResponse, PATH_GET_RUN)
|
|
363
359
|
if res is None:
|
|
364
|
-
return Run(run_id
|
|
365
|
-
|
|
366
|
-
return
|
|
367
|
-
run_id,
|
|
368
|
-
res.run.fab_id,
|
|
369
|
-
res.run.fab_version,
|
|
370
|
-
res.run.fab_hash,
|
|
371
|
-
user_config_from_proto(res.run.override_config),
|
|
372
|
-
)
|
|
360
|
+
return Run.create_empty(run_id)
|
|
361
|
+
|
|
362
|
+
return run_from_proto(res.run)
|
|
373
363
|
|
|
374
364
|
def get_fab(fab_hash: str) -> Fab:
|
|
375
365
|
# Construct the request
|
flwr/client/supernode/app.py
CHANGED
|
@@ -28,8 +28,10 @@ from cryptography.hazmat.primitives.serialization import (
|
|
|
28
28
|
)
|
|
29
29
|
|
|
30
30
|
from flwr.common import EventType, event
|
|
31
|
+
from flwr.common.args import try_obtain_root_certificates
|
|
31
32
|
from flwr.common.config import parse_config_args
|
|
32
33
|
from flwr.common.constant import (
|
|
34
|
+
CLIENTAPPIO_API_DEFAULT_SERVER_ADDRESS,
|
|
33
35
|
FLEET_API_GRPC_RERE_DEFAULT_ADDRESS,
|
|
34
36
|
ISOLATION_MODE_PROCESS,
|
|
35
37
|
ISOLATION_MODE_SUBPROCESS,
|
|
@@ -61,10 +63,21 @@ def run_supernode() -> None:
|
|
|
61
63
|
"Ignoring `--flwr-dir`.",
|
|
62
64
|
)
|
|
63
65
|
|
|
64
|
-
|
|
66
|
+
# Exit if unsupported argument is passed by the user
|
|
67
|
+
if args.app is not None:
|
|
68
|
+
log(
|
|
69
|
+
ERROR,
|
|
70
|
+
"The `app` argument is deprecated. The SuperNode now automatically "
|
|
71
|
+
"uses the ClientApp delivered from the SuperLink. Providing the app "
|
|
72
|
+
"directory manually is no longer supported. Please remove the `app` "
|
|
73
|
+
"argument from your command.",
|
|
74
|
+
)
|
|
75
|
+
sys.exit(1)
|
|
76
|
+
|
|
77
|
+
root_certificates = try_obtain_root_certificates(args, args.superlink)
|
|
65
78
|
load_fn = get_load_client_app_fn(
|
|
66
79
|
default_app_ref="",
|
|
67
|
-
app_path=
|
|
80
|
+
app_path=None,
|
|
68
81
|
flwr_dir=args.flwr_dir,
|
|
69
82
|
multi_app=True,
|
|
70
83
|
)
|
|
@@ -86,7 +99,7 @@ def run_supernode() -> None:
|
|
|
86
99
|
),
|
|
87
100
|
flwr_path=args.flwr_dir,
|
|
88
101
|
isolation=args.isolation,
|
|
89
|
-
|
|
102
|
+
clientappio_api_address=args.clientappio_api_address,
|
|
90
103
|
)
|
|
91
104
|
|
|
92
105
|
# Graceful shutdown
|
|
@@ -126,41 +139,6 @@ def _warn_deprecated_server_arg(args: argparse.Namespace) -> None:
|
|
|
126
139
|
args.superlink = args.server
|
|
127
140
|
|
|
128
141
|
|
|
129
|
-
def _get_certificates(args: argparse.Namespace) -> Optional[bytes]:
|
|
130
|
-
"""Load certificates if specified in args."""
|
|
131
|
-
# Obtain certificates
|
|
132
|
-
if args.insecure:
|
|
133
|
-
if args.root_certificates is not None:
|
|
134
|
-
sys.exit(
|
|
135
|
-
"Conflicting options: The '--insecure' flag disables HTTPS, "
|
|
136
|
-
"but '--root-certificates' was also specified. Please remove "
|
|
137
|
-
"the '--root-certificates' option when running in insecure mode, "
|
|
138
|
-
"or omit '--insecure' to use HTTPS."
|
|
139
|
-
)
|
|
140
|
-
log(
|
|
141
|
-
WARN,
|
|
142
|
-
"Option `--insecure` was set. "
|
|
143
|
-
"Starting insecure HTTP client connected to %s.",
|
|
144
|
-
args.superlink,
|
|
145
|
-
)
|
|
146
|
-
root_certificates = None
|
|
147
|
-
else:
|
|
148
|
-
# Load the certificates if provided, or load the system certificates
|
|
149
|
-
cert_path = args.root_certificates
|
|
150
|
-
if cert_path is None:
|
|
151
|
-
root_certificates = None
|
|
152
|
-
else:
|
|
153
|
-
root_certificates = Path(cert_path).read_bytes()
|
|
154
|
-
log(
|
|
155
|
-
DEBUG,
|
|
156
|
-
"Starting secure HTTPS client connected to %s "
|
|
157
|
-
"with the following certificates: %s.",
|
|
158
|
-
args.superlink,
|
|
159
|
-
cert_path,
|
|
160
|
-
)
|
|
161
|
-
return root_certificates
|
|
162
|
-
|
|
163
|
-
|
|
164
142
|
def _parse_args_run_supernode() -> argparse.ArgumentParser:
|
|
165
143
|
"""Parse flower-supernode command line arguments."""
|
|
166
144
|
parser = argparse.ArgumentParser(
|
|
@@ -171,12 +149,12 @@ def _parse_args_run_supernode() -> argparse.ArgumentParser:
|
|
|
171
149
|
"app",
|
|
172
150
|
nargs="?",
|
|
173
151
|
default=None,
|
|
174
|
-
help=
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
152
|
+
help=(
|
|
153
|
+
"(REMOVED) This argument is removed. The SuperNode now automatically "
|
|
154
|
+
"uses the ClientApp delivered from the SuperLink, so there is no need to "
|
|
155
|
+
"provide the app directory manually. This argument will be removed in a "
|
|
156
|
+
"future version."
|
|
157
|
+
),
|
|
180
158
|
)
|
|
181
159
|
_parse_args_common(parser)
|
|
182
160
|
parser.add_argument(
|
|
@@ -192,22 +170,22 @@ def _parse_args_run_supernode() -> argparse.ArgumentParser:
|
|
|
192
170
|
)
|
|
193
171
|
parser.add_argument(
|
|
194
172
|
"--isolation",
|
|
195
|
-
default=
|
|
173
|
+
default=ISOLATION_MODE_SUBPROCESS,
|
|
196
174
|
required=False,
|
|
197
175
|
choices=[
|
|
198
176
|
ISOLATION_MODE_SUBPROCESS,
|
|
199
177
|
ISOLATION_MODE_PROCESS,
|
|
200
178
|
],
|
|
201
|
-
help="Isolation mode when running a `ClientApp` (
|
|
202
|
-
"`subprocess`, `process`).
|
|
203
|
-
"
|
|
204
|
-
"
|
|
205
|
-
"independent process gets created outside of SuperNode.",
|
|
179
|
+
help="Isolation mode when running a `ClientApp` (`subprocess` by default, "
|
|
180
|
+
"possible values: `subprocess`, `process`). Use `subprocess` to configure "
|
|
181
|
+
"SuperNode to run a `ClientApp` in a subprocess. Use `process` to indicate "
|
|
182
|
+
"that a separate independent process gets created outside of SuperNode.",
|
|
206
183
|
)
|
|
207
184
|
parser.add_argument(
|
|
208
|
-
"--
|
|
209
|
-
default=
|
|
210
|
-
help="
|
|
185
|
+
"--clientappio-api-address",
|
|
186
|
+
default=CLIENTAPPIO_API_DEFAULT_SERVER_ADDRESS,
|
|
187
|
+
help="ClientAppIo API (gRPC) server address (IPv4, IPv6, or a domain name). "
|
|
188
|
+
f"By default, it is set to {CLIENTAPPIO_API_DEFAULT_SERVER_ADDRESS}.",
|
|
211
189
|
)
|
|
212
190
|
|
|
213
191
|
return parser
|
flwr/common/args.py
CHANGED
|
@@ -16,11 +16,16 @@
|
|
|
16
16
|
|
|
17
17
|
import argparse
|
|
18
18
|
import sys
|
|
19
|
-
from logging import DEBUG, WARN
|
|
19
|
+
from logging import DEBUG, ERROR, WARN
|
|
20
20
|
from os.path import isfile
|
|
21
21
|
from pathlib import Path
|
|
22
22
|
from typing import Optional
|
|
23
23
|
|
|
24
|
+
from flwr.common.constant import (
|
|
25
|
+
TRANSPORT_TYPE_GRPC_ADAPTER,
|
|
26
|
+
TRANSPORT_TYPE_GRPC_RERE,
|
|
27
|
+
TRANSPORT_TYPE_REST,
|
|
28
|
+
)
|
|
24
29
|
from flwr.common.logger import log
|
|
25
30
|
|
|
26
31
|
|
|
@@ -44,21 +49,16 @@ def add_args_flwr_app_common(parser: argparse.ArgumentParser) -> None:
|
|
|
44
49
|
"paths are provided. By default, the server runs with HTTPS enabled. "
|
|
45
50
|
"Use this flag only if you understand the risks.",
|
|
46
51
|
)
|
|
47
|
-
parser.add_argument(
|
|
48
|
-
"--root-certificates",
|
|
49
|
-
metavar="ROOT_CERT",
|
|
50
|
-
type=str,
|
|
51
|
-
help="Specifies the path to the PEM-encoded root certificate file for "
|
|
52
|
-
"establishing secure HTTPS connections.",
|
|
53
|
-
)
|
|
54
52
|
|
|
55
53
|
|
|
56
|
-
def
|
|
54
|
+
def try_obtain_root_certificates(
|
|
57
55
|
args: argparse.Namespace,
|
|
56
|
+
grpc_server_address: str,
|
|
58
57
|
) -> Optional[bytes]:
|
|
59
58
|
"""Validate and return the root certificates."""
|
|
59
|
+
root_cert_path = args.root_certificates
|
|
60
60
|
if args.insecure:
|
|
61
|
-
if
|
|
61
|
+
if root_cert_path is not None:
|
|
62
62
|
sys.exit(
|
|
63
63
|
"Conflicting options: The '--insecure' flag disables HTTPS, "
|
|
64
64
|
"but '--root-certificates' was also specified. Please remove "
|
|
@@ -67,17 +67,86 @@ def try_obtain_certificates(
|
|
|
67
67
|
)
|
|
68
68
|
log(
|
|
69
69
|
WARN,
|
|
70
|
-
"Option `--insecure` was set. Starting insecure HTTP channel.",
|
|
70
|
+
"Option `--insecure` was set. Starting insecure HTTP channel to %s.",
|
|
71
|
+
grpc_server_address,
|
|
71
72
|
)
|
|
72
73
|
root_certificates = None
|
|
73
74
|
else:
|
|
74
75
|
# Load the certificates if provided, or load the system certificates
|
|
75
|
-
if
|
|
76
|
-
|
|
77
|
-
|
|
76
|
+
if root_cert_path is None:
|
|
77
|
+
log(
|
|
78
|
+
WARN,
|
|
79
|
+
"Both `--insecure` and `--root-certificates` were not set. "
|
|
80
|
+
"Using system certificates.",
|
|
81
|
+
)
|
|
82
|
+
root_certificates = None
|
|
83
|
+
elif not isfile(root_cert_path):
|
|
84
|
+
log(ERROR, "Path argument `--root-certificates` does not point to a file.")
|
|
85
|
+
sys.exit(1)
|
|
86
|
+
else:
|
|
87
|
+
root_certificates = Path(root_cert_path).read_bytes()
|
|
78
88
|
log(
|
|
79
89
|
DEBUG,
|
|
80
|
-
"Starting secure HTTPS channel
|
|
81
|
-
|
|
90
|
+
"Starting secure HTTPS channel to %s "
|
|
91
|
+
"with the following certificates: %s.",
|
|
92
|
+
grpc_server_address,
|
|
93
|
+
root_cert_path,
|
|
82
94
|
)
|
|
83
95
|
return root_certificates
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def try_obtain_server_certificates(
|
|
99
|
+
args: argparse.Namespace,
|
|
100
|
+
transport_type: str,
|
|
101
|
+
) -> Optional[tuple[bytes, bytes, bytes]]:
|
|
102
|
+
"""Validate and return the CA cert, server cert, and server private key."""
|
|
103
|
+
if args.insecure:
|
|
104
|
+
log(WARN, "Option `--insecure` was set. Starting insecure HTTP server.")
|
|
105
|
+
return None
|
|
106
|
+
# Check if certificates are provided
|
|
107
|
+
if transport_type in [TRANSPORT_TYPE_GRPC_RERE, TRANSPORT_TYPE_GRPC_ADAPTER]:
|
|
108
|
+
if args.ssl_certfile and args.ssl_keyfile and args.ssl_ca_certfile:
|
|
109
|
+
if not isfile(args.ssl_ca_certfile):
|
|
110
|
+
sys.exit("Path argument `--ssl-ca-certfile` does not point to a file.")
|
|
111
|
+
if not isfile(args.ssl_certfile):
|
|
112
|
+
sys.exit("Path argument `--ssl-certfile` does not point to a file.")
|
|
113
|
+
if not isfile(args.ssl_keyfile):
|
|
114
|
+
sys.exit("Path argument `--ssl-keyfile` does not point to a file.")
|
|
115
|
+
certificates = (
|
|
116
|
+
Path(args.ssl_ca_certfile).read_bytes(), # CA certificate
|
|
117
|
+
Path(args.ssl_certfile).read_bytes(), # server certificate
|
|
118
|
+
Path(args.ssl_keyfile).read_bytes(), # server private key
|
|
119
|
+
)
|
|
120
|
+
return certificates
|
|
121
|
+
if args.ssl_certfile or args.ssl_keyfile or args.ssl_ca_certfile:
|
|
122
|
+
sys.exit(
|
|
123
|
+
"You need to provide valid file paths to `--ssl-certfile`, "
|
|
124
|
+
"`--ssl-keyfile`, and `—-ssl-ca-certfile` to create a secure "
|
|
125
|
+
"connection in Fleet API server (gRPC-rere)."
|
|
126
|
+
)
|
|
127
|
+
if transport_type == TRANSPORT_TYPE_REST:
|
|
128
|
+
if args.ssl_certfile and args.ssl_keyfile:
|
|
129
|
+
if not isfile(args.ssl_certfile):
|
|
130
|
+
sys.exit("Path argument `--ssl-certfile` does not point to a file.")
|
|
131
|
+
if not isfile(args.ssl_keyfile):
|
|
132
|
+
sys.exit("Path argument `--ssl-keyfile` does not point to a file.")
|
|
133
|
+
certificates = (
|
|
134
|
+
b"",
|
|
135
|
+
Path(args.ssl_certfile).read_bytes(), # server certificate
|
|
136
|
+
Path(args.ssl_keyfile).read_bytes(), # server private key
|
|
137
|
+
)
|
|
138
|
+
return certificates
|
|
139
|
+
if args.ssl_certfile or args.ssl_keyfile:
|
|
140
|
+
sys.exit(
|
|
141
|
+
"You need to provide valid file paths to `--ssl-certfile` "
|
|
142
|
+
"and `--ssl-keyfile` to create a secure connection "
|
|
143
|
+
"in Fleet API server (REST, experimental)."
|
|
144
|
+
)
|
|
145
|
+
log(
|
|
146
|
+
ERROR,
|
|
147
|
+
"Certificates are required unless running in insecure mode. "
|
|
148
|
+
"Please provide certificate paths to `--ssl-certfile`, "
|
|
149
|
+
"`--ssl-keyfile`, and `—-ssl-ca-certfile` or run the server "
|
|
150
|
+
"in insecure mode using '--insecure' if you understand the risks.",
|
|
151
|
+
)
|
|
152
|
+
sys.exit(1)
|
flwr/common/constant.py
CHANGED
|
@@ -38,17 +38,30 @@ TRANSPORT_TYPES = [
|
|
|
38
38
|
]
|
|
39
39
|
|
|
40
40
|
# Addresses
|
|
41
|
+
# Ports
|
|
42
|
+
CLIENTAPPIO_PORT = "9094"
|
|
43
|
+
SERVERAPPIO_PORT = "9091"
|
|
44
|
+
FLEETAPI_GRPC_RERE_PORT = "9092"
|
|
45
|
+
FLEETAPI_PORT = "9095"
|
|
46
|
+
EXEC_API_PORT = "9093"
|
|
47
|
+
SIMULATIONIO_PORT = "9096"
|
|
48
|
+
# Octets
|
|
49
|
+
SERVER_OCTET = "0.0.0.0"
|
|
50
|
+
CLIENT_OCTET = "127.0.0.1"
|
|
41
51
|
# SuperNode
|
|
42
|
-
|
|
52
|
+
CLIENTAPPIO_API_DEFAULT_SERVER_ADDRESS = f"{SERVER_OCTET}:{CLIENTAPPIO_PORT}"
|
|
53
|
+
CLIENTAPPIO_API_DEFAULT_CLIENT_ADDRESS = f"{CLIENT_OCTET}:{CLIENTAPPIO_PORT}"
|
|
43
54
|
# SuperLink
|
|
44
|
-
|
|
45
|
-
|
|
55
|
+
SERVERAPPIO_API_DEFAULT_SERVER_ADDRESS = f"{SERVER_OCTET}:{SERVERAPPIO_PORT}"
|
|
56
|
+
SERVERAPPIO_API_DEFAULT_CLIENT_ADDRESS = f"{CLIENT_OCTET}:{SERVERAPPIO_PORT}"
|
|
57
|
+
FLEET_API_GRPC_RERE_DEFAULT_ADDRESS = f"{SERVER_OCTET}:{FLEETAPI_GRPC_RERE_PORT}"
|
|
46
58
|
FLEET_API_GRPC_BIDI_DEFAULT_ADDRESS = (
|
|
47
59
|
"[::]:8080" # IPv6 to keep start_server compatible
|
|
48
60
|
)
|
|
49
|
-
FLEET_API_REST_DEFAULT_ADDRESS = "
|
|
50
|
-
|
|
51
|
-
|
|
61
|
+
FLEET_API_REST_DEFAULT_ADDRESS = f"{SERVER_OCTET}:{FLEETAPI_PORT}"
|
|
62
|
+
EXEC_API_DEFAULT_SERVER_ADDRESS = f"{SERVER_OCTET}:{EXEC_API_PORT}"
|
|
63
|
+
SIMULATIONIO_API_DEFAULT_SERVER_ADDRESS = f"{SERVER_OCTET}:{SIMULATIONIO_PORT}"
|
|
64
|
+
SIMULATIONIO_API_DEFAULT_CLIENT_ADDRESS = f"{CLIENT_OCTET}:{SIMULATIONIO_PORT}"
|
|
52
65
|
|
|
53
66
|
# Constants for ping
|
|
54
67
|
PING_DEFAULT_INTERVAL = 30
|
|
@@ -94,6 +107,9 @@ CONN_RECONNECT_INTERVAL = 0.5 # Reconnect interval between two stream connectio
|
|
|
94
107
|
LOG_STREAM_INTERVAL = 0.5 # Log stream interval for `ExecServicer.StreamLogs`
|
|
95
108
|
LOG_UPLOAD_INTERVAL = 0.2 # Minimum interval between two log uploads
|
|
96
109
|
|
|
110
|
+
# Retry configurations
|
|
111
|
+
MAX_RETRY_DELAY = 20 # Maximum delay duration between two consecutive retries.
|
|
112
|
+
|
|
97
113
|
|
|
98
114
|
class MessageType:
|
|
99
115
|
"""Message type."""
|
|
@@ -134,6 +150,8 @@ class ErrorCode:
|
|
|
134
150
|
UNKNOWN = 0
|
|
135
151
|
LOAD_CLIENT_APP_EXCEPTION = 1
|
|
136
152
|
CLIENT_APP_RAISED_EXCEPTION = 2
|
|
153
|
+
MESSAGE_UNAVAILABLE = 3
|
|
154
|
+
REPLY_MESSAGE_UNAVAILABLE = 4
|
|
137
155
|
|
|
138
156
|
def __new__(cls) -> ErrorCode:
|
|
139
157
|
"""Prevent instantiation."""
|
flwr/common/date.py
CHANGED
|
@@ -21,3 +21,21 @@ import datetime
|
|
|
21
21
|
def now() -> datetime.datetime:
|
|
22
22
|
"""Construct a datetime from time.time() with time zone set to UTC."""
|
|
23
23
|
return datetime.datetime.now(tz=datetime.timezone.utc)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def format_timedelta(td: datetime.timedelta) -> str:
|
|
27
|
+
"""Format a timedelta as a string."""
|
|
28
|
+
days = td.days
|
|
29
|
+
hours, remainder = divmod(td.seconds, 3600)
|
|
30
|
+
minutes, seconds = divmod(remainder, 60)
|
|
31
|
+
|
|
32
|
+
if days > 0:
|
|
33
|
+
return f"{days}d {hours:02}:{minutes:02}:{seconds:02}"
|
|
34
|
+
return f"{hours:02}:{minutes:02}:{seconds:02}"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def isoformat8601_utc(dt: datetime.datetime) -> str:
|
|
38
|
+
"""Return the datetime formatted as an ISO 8601 string with a trailing 'Z'."""
|
|
39
|
+
if dt.tzinfo != datetime.timezone.utc:
|
|
40
|
+
raise ValueError("Expected datetime with timezone set to UTC")
|
|
41
|
+
return dt.isoformat(timespec="seconds").replace("+00:00", "Z")
|
flwr/common/grpc.py
CHANGED
|
@@ -53,7 +53,10 @@ def create_channel(
|
|
|
53
53
|
channel = grpc.insecure_channel(server_address, options=channel_options)
|
|
54
54
|
log(DEBUG, "Opened insecure gRPC connection (no certificates were passed)")
|
|
55
55
|
else:
|
|
56
|
-
|
|
56
|
+
try:
|
|
57
|
+
ssl_channel_credentials = grpc.ssl_channel_credentials(root_certificates)
|
|
58
|
+
except Exception as e:
|
|
59
|
+
raise ValueError(f"Failed to create SSL channel credentials: {e}") from e
|
|
57
60
|
channel = grpc.secure_channel(
|
|
58
61
|
server_address, ssl_channel_credentials, options=channel_options
|
|
59
62
|
)
|
flwr/common/serde.py
CHANGED
|
@@ -872,6 +872,11 @@ def run_to_proto(run: typing.Run) -> ProtoRun:
|
|
|
872
872
|
fab_version=run.fab_version,
|
|
873
873
|
fab_hash=run.fab_hash,
|
|
874
874
|
override_config=user_config_to_proto(run.override_config),
|
|
875
|
+
pending_at=run.pending_at,
|
|
876
|
+
starting_at=run.starting_at,
|
|
877
|
+
running_at=run.running_at,
|
|
878
|
+
finished_at=run.finished_at,
|
|
879
|
+
status=run_status_to_proto(run.status),
|
|
875
880
|
)
|
|
876
881
|
return proto
|
|
877
882
|
|
|
@@ -884,6 +889,11 @@ def run_from_proto(run_proto: ProtoRun) -> typing.Run:
|
|
|
884
889
|
fab_version=run_proto.fab_version,
|
|
885
890
|
fab_hash=run_proto.fab_hash,
|
|
886
891
|
override_config=user_config_from_proto(run_proto.override_config),
|
|
892
|
+
pending_at=run_proto.pending_at,
|
|
893
|
+
starting_at=run_proto.starting_at,
|
|
894
|
+
running_at=run_proto.running_at,
|
|
895
|
+
finished_at=run_proto.finished_at,
|
|
896
|
+
status=run_status_from_proto(run_proto.status),
|
|
887
897
|
)
|
|
888
898
|
return run
|
|
889
899
|
|
flwr/common/typing.py
CHANGED
|
@@ -208,7 +208,16 @@ class ClientMessage:
|
|
|
208
208
|
|
|
209
209
|
|
|
210
210
|
@dataclass
|
|
211
|
-
class
|
|
211
|
+
class RunStatus:
|
|
212
|
+
"""Run status information."""
|
|
213
|
+
|
|
214
|
+
status: str
|
|
215
|
+
sub_status: str
|
|
216
|
+
details: str
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
@dataclass
|
|
220
|
+
class Run: # pylint: disable=too-many-instance-attributes
|
|
212
221
|
"""Run details."""
|
|
213
222
|
|
|
214
223
|
run_id: int
|
|
@@ -216,15 +225,27 @@ class Run:
|
|
|
216
225
|
fab_version: str
|
|
217
226
|
fab_hash: str
|
|
218
227
|
override_config: UserConfig
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
+
pending_at: str
|
|
229
|
+
starting_at: str
|
|
230
|
+
running_at: str
|
|
231
|
+
finished_at: str
|
|
232
|
+
status: RunStatus
|
|
233
|
+
|
|
234
|
+
@classmethod
|
|
235
|
+
def create_empty(cls, run_id: int) -> "Run":
|
|
236
|
+
"""Return an empty Run instance."""
|
|
237
|
+
return cls(
|
|
238
|
+
run_id=run_id,
|
|
239
|
+
fab_id="",
|
|
240
|
+
fab_version="",
|
|
241
|
+
fab_hash="",
|
|
242
|
+
override_config={},
|
|
243
|
+
pending_at="",
|
|
244
|
+
starting_at="",
|
|
245
|
+
running_at="",
|
|
246
|
+
finished_at="",
|
|
247
|
+
status=RunStatus(status="", sub_status="", details=""),
|
|
248
|
+
)
|
|
228
249
|
|
|
229
250
|
|
|
230
251
|
@dataclass
|