flwr-nightly 1.11.0.dev20240813__py3-none-any.whl → 1.11.0.dev20240822__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flwr-nightly might be problematic. Click here for more details.
- flwr/cli/config_utils.py +2 -2
- flwr/cli/install.py +3 -1
- flwr/cli/run/run.py +15 -11
- flwr/client/app.py +132 -14
- flwr/client/clientapp/__init__.py +22 -0
- flwr/client/clientapp/app.py +233 -0
- flwr/client/clientapp/clientappio_servicer.py +244 -0
- flwr/client/clientapp/utils.py +108 -0
- flwr/client/grpc_rere_client/connection.py +9 -1
- flwr/client/node_state.py +17 -4
- flwr/client/rest_client/connection.py +16 -3
- flwr/client/supernode/__init__.py +0 -2
- flwr/client/supernode/app.py +36 -164
- flwr/common/__init__.py +4 -0
- flwr/common/config.py +31 -10
- flwr/common/record/configsrecord.py +49 -15
- flwr/common/record/metricsrecord.py +54 -14
- flwr/common/record/parametersrecord.py +84 -17
- flwr/common/record/recordset.py +80 -8
- flwr/common/record/typeddict.py +20 -58
- flwr/common/recordset_compat.py +6 -6
- flwr/common/serde.py +24 -2
- flwr/common/typing.py +1 -0
- flwr/proto/clientappio_pb2.py +17 -13
- flwr/proto/clientappio_pb2.pyi +24 -2
- flwr/proto/clientappio_pb2_grpc.py +34 -0
- flwr/proto/clientappio_pb2_grpc.pyi +13 -0
- flwr/proto/exec_pb2.py +16 -15
- flwr/proto/exec_pb2.pyi +7 -4
- flwr/proto/message_pb2.py +2 -2
- flwr/proto/message_pb2.pyi +4 -1
- flwr/server/app.py +15 -0
- flwr/server/driver/grpc_driver.py +1 -0
- flwr/server/run_serverapp.py +18 -2
- flwr/server/server.py +3 -1
- flwr/server/superlink/driver/driver_grpc.py +3 -0
- flwr/server/superlink/driver/driver_servicer.py +32 -4
- flwr/server/superlink/ffs/disk_ffs.py +6 -3
- flwr/server/superlink/ffs/ffs.py +3 -3
- flwr/server/superlink/ffs/ffs_factory.py +47 -0
- flwr/server/superlink/fleet/grpc_rere/fleet_servicer.py +12 -4
- flwr/server/superlink/fleet/grpc_rere/server_interceptor.py +8 -2
- flwr/server/superlink/fleet/message_handler/message_handler.py +16 -1
- flwr/server/superlink/fleet/vce/backend/raybackend.py +5 -2
- flwr/server/superlink/fleet/vce/vce_api.py +2 -2
- flwr/server/superlink/state/in_memory_state.py +7 -5
- flwr/server/superlink/state/sqlite_state.py +17 -7
- flwr/server/superlink/state/state.py +4 -3
- flwr/server/workflow/default_workflows.py +3 -1
- flwr/simulation/run_simulation.py +5 -67
- flwr/superexec/app.py +3 -3
- flwr/superexec/deployment.py +8 -9
- flwr/superexec/exec_servicer.py +1 -1
- {flwr_nightly-1.11.0.dev20240813.dist-info → flwr_nightly-1.11.0.dev20240822.dist-info}/METADATA +2 -2
- {flwr_nightly-1.11.0.dev20240813.dist-info → flwr_nightly-1.11.0.dev20240822.dist-info}/RECORD +58 -53
- {flwr_nightly-1.11.0.dev20240813.dist-info → flwr_nightly-1.11.0.dev20240822.dist-info}/entry_points.txt +1 -1
- {flwr_nightly-1.11.0.dev20240813.dist-info → flwr_nightly-1.11.0.dev20240822.dist-info}/LICENSE +0 -0
- {flwr_nightly-1.11.0.dev20240813.dist-info → flwr_nightly-1.11.0.dev20240822.dist-info}/WHEEL +0 -0
flwr/client/supernode/app.py
CHANGED
|
@@ -16,9 +16,9 @@
|
|
|
16
16
|
|
|
17
17
|
import argparse
|
|
18
18
|
import sys
|
|
19
|
-
from logging import DEBUG, INFO, WARN
|
|
19
|
+
from logging import DEBUG, ERROR, INFO, WARN
|
|
20
20
|
from pathlib import Path
|
|
21
|
-
from typing import
|
|
21
|
+
from typing import Optional, Tuple
|
|
22
22
|
|
|
23
23
|
from cryptography.exceptions import UnsupportedAlgorithm
|
|
24
24
|
from cryptography.hazmat.primitives.asymmetric import ec
|
|
@@ -27,15 +27,8 @@ from cryptography.hazmat.primitives.serialization import (
|
|
|
27
27
|
load_ssh_public_key,
|
|
28
28
|
)
|
|
29
29
|
|
|
30
|
-
from flwr.client.client_app import ClientApp, LoadClientAppError
|
|
31
30
|
from flwr.common import EventType, event
|
|
32
|
-
from flwr.common.config import
|
|
33
|
-
get_flwr_dir,
|
|
34
|
-
get_metadata_from_config,
|
|
35
|
-
get_project_config,
|
|
36
|
-
get_project_dir,
|
|
37
|
-
parse_config_args,
|
|
38
|
-
)
|
|
31
|
+
from flwr.common.config import parse_config_args
|
|
39
32
|
from flwr.common.constant import (
|
|
40
33
|
TRANSPORT_TYPE_GRPC_ADAPTER,
|
|
41
34
|
TRANSPORT_TYPE_GRPC_RERE,
|
|
@@ -43,9 +36,13 @@ from flwr.common.constant import (
|
|
|
43
36
|
)
|
|
44
37
|
from flwr.common.exit_handlers import register_exit_handlers
|
|
45
38
|
from flwr.common.logger import log, warn_deprecated_feature
|
|
46
|
-
from flwr.common.object_ref import load_app, validate
|
|
47
39
|
|
|
48
|
-
from ..app import
|
|
40
|
+
from ..app import (
|
|
41
|
+
ISOLATION_MODE_PROCESS,
|
|
42
|
+
ISOLATION_MODE_SUBPROCESS,
|
|
43
|
+
start_client_internal,
|
|
44
|
+
)
|
|
45
|
+
from ..clientapp.utils import get_load_client_app_fn
|
|
49
46
|
|
|
50
47
|
ADDRESS_FLEET_API_GRPC_RERE = "0.0.0.0:9092"
|
|
51
48
|
|
|
@@ -61,7 +58,7 @@ def run_supernode() -> None:
|
|
|
61
58
|
_warn_deprecated_server_arg(args)
|
|
62
59
|
|
|
63
60
|
root_certificates = _get_certificates(args)
|
|
64
|
-
load_fn =
|
|
61
|
+
load_fn = get_load_client_app_fn(
|
|
65
62
|
default_app_ref="",
|
|
66
63
|
app_path=args.app,
|
|
67
64
|
flwr_dir=args.flwr_dir,
|
|
@@ -69,7 +66,9 @@ def run_supernode() -> None:
|
|
|
69
66
|
)
|
|
70
67
|
authentication_keys = _try_setup_client_authentication(args)
|
|
71
68
|
|
|
72
|
-
|
|
69
|
+
log(DEBUG, "Isolation mode: %s", args.isolation)
|
|
70
|
+
|
|
71
|
+
start_client_internal(
|
|
73
72
|
server_address=args.superlink,
|
|
74
73
|
load_client_app_fn=load_fn,
|
|
75
74
|
transport=args.transport,
|
|
@@ -79,7 +78,8 @@ def run_supernode() -> None:
|
|
|
79
78
|
max_retries=args.max_retries,
|
|
80
79
|
max_wait_time=args.max_wait_time,
|
|
81
80
|
node_config=parse_config_args([args.node_config]),
|
|
82
|
-
|
|
81
|
+
isolation=args.isolation,
|
|
82
|
+
supernode_address=args.supernode_address,
|
|
83
83
|
)
|
|
84
84
|
|
|
85
85
|
# Graceful shutdown
|
|
@@ -90,59 +90,13 @@ def run_supernode() -> None:
|
|
|
90
90
|
|
|
91
91
|
def run_client_app() -> None:
|
|
92
92
|
"""Run Flower client app."""
|
|
93
|
-
log(INFO, "Long-running Flower client starting")
|
|
94
|
-
|
|
95
93
|
event(EventType.RUN_CLIENT_APP_ENTER)
|
|
96
|
-
|
|
97
|
-
args = _parse_args_run_client_app().parse_args()
|
|
98
|
-
|
|
99
|
-
_warn_deprecated_server_arg(args)
|
|
100
|
-
|
|
101
|
-
root_certificates = _get_certificates(args)
|
|
102
|
-
load_fn = _get_load_client_app_fn(
|
|
103
|
-
default_app_ref=getattr(args, "client-app"),
|
|
104
|
-
app_path=args.dir,
|
|
105
|
-
multi_app=False,
|
|
106
|
-
)
|
|
107
|
-
authentication_keys = _try_setup_client_authentication(args)
|
|
108
|
-
|
|
109
|
-
_start_client_internal(
|
|
110
|
-
server_address=args.superlink,
|
|
111
|
-
node_config=parse_config_args([args.node_config]),
|
|
112
|
-
load_client_app_fn=load_fn,
|
|
113
|
-
transport=args.transport,
|
|
114
|
-
root_certificates=root_certificates,
|
|
115
|
-
insecure=args.insecure,
|
|
116
|
-
authentication_keys=authentication_keys,
|
|
117
|
-
max_retries=args.max_retries,
|
|
118
|
-
max_wait_time=args.max_wait_time,
|
|
119
|
-
)
|
|
120
|
-
register_exit_handlers(event_type=EventType.RUN_CLIENT_APP_LEAVE)
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
def flwr_clientapp() -> None:
|
|
124
|
-
"""Run process-isolated Flower ClientApp."""
|
|
125
|
-
log(INFO, "Starting Flower ClientApp")
|
|
126
|
-
|
|
127
|
-
parser = argparse.ArgumentParser(
|
|
128
|
-
description="Run a Flower ClientApp",
|
|
129
|
-
)
|
|
130
|
-
parser.add_argument(
|
|
131
|
-
"--address",
|
|
132
|
-
help="Address of SuperNode ClientAppIo gRPC servicer",
|
|
133
|
-
)
|
|
134
|
-
parser.add_argument(
|
|
135
|
-
"--token",
|
|
136
|
-
help="Unique token generated by SuperNode for each ClientApp execution",
|
|
137
|
-
)
|
|
138
|
-
args = parser.parse_args()
|
|
139
94
|
log(
|
|
140
|
-
|
|
141
|
-
"
|
|
142
|
-
"with the token %s",
|
|
143
|
-
args.address,
|
|
144
|
-
args.token,
|
|
95
|
+
ERROR,
|
|
96
|
+
"The command `flower-client-app` has been replaced by `flower-supernode`.",
|
|
145
97
|
)
|
|
98
|
+
log(INFO, "Execute `flower-supernode --help` to learn how to use it.")
|
|
99
|
+
register_exit_handlers(event_type=EventType.RUN_CLIENT_APP_LEAVE)
|
|
146
100
|
|
|
147
101
|
|
|
148
102
|
def _warn_deprecated_server_arg(args: argparse.Namespace) -> None:
|
|
@@ -200,85 +154,6 @@ def _get_certificates(args: argparse.Namespace) -> Optional[bytes]:
|
|
|
200
154
|
return root_certificates
|
|
201
155
|
|
|
202
156
|
|
|
203
|
-
def _get_load_client_app_fn(
|
|
204
|
-
default_app_ref: str,
|
|
205
|
-
app_path: Optional[str],
|
|
206
|
-
multi_app: bool,
|
|
207
|
-
flwr_dir: Optional[str] = None,
|
|
208
|
-
) -> Callable[[str, str], ClientApp]:
|
|
209
|
-
"""Get the load_client_app_fn function.
|
|
210
|
-
|
|
211
|
-
If `multi_app` is True, this function loads the specified ClientApp
|
|
212
|
-
based on `fab_id` and `fab_version`. If `fab_id` is empty, a default
|
|
213
|
-
ClientApp will be loaded.
|
|
214
|
-
|
|
215
|
-
If `multi_app` is False, it ignores `fab_id` and `fab_version` and
|
|
216
|
-
loads a default ClientApp.
|
|
217
|
-
"""
|
|
218
|
-
if not multi_app:
|
|
219
|
-
log(
|
|
220
|
-
DEBUG,
|
|
221
|
-
"Flower SuperNode will load and validate ClientApp `%s`",
|
|
222
|
-
default_app_ref,
|
|
223
|
-
)
|
|
224
|
-
|
|
225
|
-
valid, error_msg = validate(default_app_ref, project_dir=app_path)
|
|
226
|
-
if not valid and error_msg:
|
|
227
|
-
raise LoadClientAppError(error_msg) from None
|
|
228
|
-
|
|
229
|
-
def _load(fab_id: str, fab_version: str) -> ClientApp:
|
|
230
|
-
runtime_app_dir = Path(app_path if app_path else "").absolute()
|
|
231
|
-
# If multi-app feature is disabled
|
|
232
|
-
if not multi_app:
|
|
233
|
-
# Set app reference
|
|
234
|
-
client_app_ref = default_app_ref
|
|
235
|
-
# If multi-app feature is enabled but app directory is provided
|
|
236
|
-
elif app_path is not None:
|
|
237
|
-
config = get_project_config(runtime_app_dir)
|
|
238
|
-
this_fab_version, this_fab_id = get_metadata_from_config(config)
|
|
239
|
-
|
|
240
|
-
if this_fab_version != fab_version or this_fab_id != fab_id:
|
|
241
|
-
raise LoadClientAppError(
|
|
242
|
-
f"FAB ID or version mismatch: Expected FAB ID '{this_fab_id}' and "
|
|
243
|
-
f"FAB version '{this_fab_version}', but received FAB ID '{fab_id}' "
|
|
244
|
-
f"and FAB version '{fab_version}'.",
|
|
245
|
-
) from None
|
|
246
|
-
|
|
247
|
-
# log(WARN, "FAB ID is not provided; the default ClientApp will be loaded.")
|
|
248
|
-
|
|
249
|
-
# Set app reference
|
|
250
|
-
client_app_ref = config["tool"]["flwr"]["app"]["components"]["clientapp"]
|
|
251
|
-
# If multi-app feature is enabled
|
|
252
|
-
else:
|
|
253
|
-
try:
|
|
254
|
-
runtime_app_dir = get_project_dir(
|
|
255
|
-
fab_id, fab_version, get_flwr_dir(flwr_dir)
|
|
256
|
-
)
|
|
257
|
-
config = get_project_config(runtime_app_dir)
|
|
258
|
-
except Exception as e:
|
|
259
|
-
raise LoadClientAppError("Failed to load ClientApp") from e
|
|
260
|
-
|
|
261
|
-
# Set app reference
|
|
262
|
-
client_app_ref = config["tool"]["flwr"]["app"]["components"]["clientapp"]
|
|
263
|
-
|
|
264
|
-
# Load ClientApp
|
|
265
|
-
log(
|
|
266
|
-
DEBUG,
|
|
267
|
-
"Loading ClientApp `%s`",
|
|
268
|
-
client_app_ref,
|
|
269
|
-
)
|
|
270
|
-
client_app = load_app(client_app_ref, LoadClientAppError, runtime_app_dir)
|
|
271
|
-
|
|
272
|
-
if not isinstance(client_app, ClientApp):
|
|
273
|
-
raise LoadClientAppError(
|
|
274
|
-
f"Attribute {client_app_ref} is not of type {ClientApp}",
|
|
275
|
-
) from None
|
|
276
|
-
|
|
277
|
-
return client_app
|
|
278
|
-
|
|
279
|
-
return _load
|
|
280
|
-
|
|
281
|
-
|
|
282
157
|
def _parse_args_run_supernode() -> argparse.ArgumentParser:
|
|
283
158
|
"""Parse flower-supernode command line arguments."""
|
|
284
159
|
parser = argparse.ArgumentParser(
|
|
@@ -308,27 +183,24 @@ def _parse_args_run_supernode() -> argparse.ArgumentParser:
|
|
|
308
183
|
- `$HOME/.flwr/` in all other cases
|
|
309
184
|
""",
|
|
310
185
|
)
|
|
311
|
-
|
|
312
|
-
return parser
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
def _parse_args_run_client_app() -> argparse.ArgumentParser:
|
|
316
|
-
"""Parse flower-client-app command line arguments."""
|
|
317
|
-
parser = argparse.ArgumentParser(
|
|
318
|
-
description="Start a Flower client app",
|
|
319
|
-
)
|
|
320
|
-
|
|
321
186
|
parser.add_argument(
|
|
322
|
-
"
|
|
323
|
-
|
|
187
|
+
"--isolation",
|
|
188
|
+
default=None,
|
|
189
|
+
required=False,
|
|
190
|
+
choices=[
|
|
191
|
+
ISOLATION_MODE_SUBPROCESS,
|
|
192
|
+
ISOLATION_MODE_PROCESS,
|
|
193
|
+
],
|
|
194
|
+
help="Isolation mode when running `ClientApp` (optional, possible values: "
|
|
195
|
+
"`subprocess`, `process`). By default, `ClientApp` runs in the same process "
|
|
196
|
+
"that executes the SuperNode. Use `subprocess` to configure SuperNode to run "
|
|
197
|
+
"`ClientApp` in a subprocess. Use `process` to indicate that a separate "
|
|
198
|
+
"independent process gets created outside of SuperNode.",
|
|
324
199
|
)
|
|
325
|
-
_parse_args_common(parser=parser)
|
|
326
200
|
parser.add_argument(
|
|
327
|
-
"--
|
|
328
|
-
default="",
|
|
329
|
-
help="
|
|
330
|
-
"app from there."
|
|
331
|
-
" Default: current working directory.",
|
|
201
|
+
"--supernode-address",
|
|
202
|
+
default="0.0.0.0:9094",
|
|
203
|
+
help="Set the SuperNode gRPC server address. Defaults to `0.0.0.0:9094`.",
|
|
332
204
|
)
|
|
333
205
|
|
|
334
206
|
return parser
|
|
@@ -410,9 +282,9 @@ def _parse_args_common(parser: argparse.ArgumentParser) -> None:
|
|
|
410
282
|
parser.add_argument(
|
|
411
283
|
"--node-config",
|
|
412
284
|
type=str,
|
|
413
|
-
help="A
|
|
285
|
+
help="A space separated list of key/value pairs (separated by `=`) to "
|
|
414
286
|
"configure the SuperNode. "
|
|
415
|
-
"E.g. --node-config 'key1=\"value1\"
|
|
287
|
+
"E.g. --node-config 'key1=\"value1\" partition-id=0 num-partitions=100'",
|
|
416
288
|
)
|
|
417
289
|
|
|
418
290
|
|
flwr/common/__init__.py
CHANGED
|
@@ -41,6 +41,7 @@ from .telemetry import event as event
|
|
|
41
41
|
from .typing import ClientMessage as ClientMessage
|
|
42
42
|
from .typing import Code as Code
|
|
43
43
|
from .typing import Config as Config
|
|
44
|
+
from .typing import ConfigsRecordValues as ConfigsRecordValues
|
|
44
45
|
from .typing import DisconnectRes as DisconnectRes
|
|
45
46
|
from .typing import EvaluateIns as EvaluateIns
|
|
46
47
|
from .typing import EvaluateRes as EvaluateRes
|
|
@@ -52,6 +53,7 @@ from .typing import GetPropertiesIns as GetPropertiesIns
|
|
|
52
53
|
from .typing import GetPropertiesRes as GetPropertiesRes
|
|
53
54
|
from .typing import Metrics as Metrics
|
|
54
55
|
from .typing import MetricsAggregationFn as MetricsAggregationFn
|
|
56
|
+
from .typing import MetricsRecordValues as MetricsRecordValues
|
|
55
57
|
from .typing import NDArray as NDArray
|
|
56
58
|
from .typing import NDArrays as NDArrays
|
|
57
59
|
from .typing import Parameters as Parameters
|
|
@@ -67,6 +69,7 @@ __all__ = [
|
|
|
67
69
|
"Code",
|
|
68
70
|
"Config",
|
|
69
71
|
"ConfigsRecord",
|
|
72
|
+
"ConfigsRecordValues",
|
|
70
73
|
"Context",
|
|
71
74
|
"DEFAULT_TTL",
|
|
72
75
|
"DisconnectRes",
|
|
@@ -88,6 +91,7 @@ __all__ = [
|
|
|
88
91
|
"Metrics",
|
|
89
92
|
"MetricsAggregationFn",
|
|
90
93
|
"MetricsRecord",
|
|
94
|
+
"MetricsRecordValues",
|
|
91
95
|
"NDArray",
|
|
92
96
|
"NDArrays",
|
|
93
97
|
"Parameters",
|
flwr/common/config.py
CHANGED
|
@@ -15,12 +15,13 @@
|
|
|
15
15
|
"""Provide functions for managing global Flower config."""
|
|
16
16
|
|
|
17
17
|
import os
|
|
18
|
+
import re
|
|
18
19
|
from pathlib import Path
|
|
19
20
|
from typing import Any, Dict, List, Optional, Tuple, Union, cast, get_args
|
|
20
21
|
|
|
21
22
|
import tomli
|
|
22
23
|
|
|
23
|
-
from flwr.cli.config_utils import validate_fields
|
|
24
|
+
from flwr.cli.config_utils import get_fab_config, validate_fields
|
|
24
25
|
from flwr.common.constant import APP_DIR, FAB_CONFIG_FILE, FLWR_HOME
|
|
25
26
|
from flwr.common.typing import Run, UserConfig, UserConfigValue
|
|
26
27
|
|
|
@@ -74,10 +75,15 @@ def get_project_config(project_dir: Union[str, Path]) -> Dict[str, Any]:
|
|
|
74
75
|
return config
|
|
75
76
|
|
|
76
77
|
|
|
77
|
-
def
|
|
78
|
+
def fuse_dicts(
|
|
78
79
|
main_dict: UserConfig,
|
|
79
80
|
override_dict: UserConfig,
|
|
80
81
|
) -> UserConfig:
|
|
82
|
+
"""Merge a config with the overrides.
|
|
83
|
+
|
|
84
|
+
Remove the nesting by adding the nested keys as prefixes separated by dots, and fuse
|
|
85
|
+
it with the override dict.
|
|
86
|
+
"""
|
|
81
87
|
fused_dict = main_dict.copy()
|
|
82
88
|
|
|
83
89
|
for key, value in override_dict.items():
|
|
@@ -96,7 +102,19 @@ def get_fused_config_from_dir(
|
|
|
96
102
|
)
|
|
97
103
|
flat_default_config = flatten_dict(default_config)
|
|
98
104
|
|
|
99
|
-
return
|
|
105
|
+
return fuse_dicts(flat_default_config, override_config)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def get_fused_config_from_fab(fab_file: Union[Path, bytes], run: Run) -> UserConfig:
|
|
109
|
+
"""Fuse default config in a `FAB` with overrides in a `Run`.
|
|
110
|
+
|
|
111
|
+
This enables obtaining a run-config without having to install the FAB. This
|
|
112
|
+
function mirrors `get_fused_config_from_dir`. This is useful when the execution
|
|
113
|
+
of the FAB is delegated to a different process.
|
|
114
|
+
"""
|
|
115
|
+
default_config = get_fab_config(fab_file)["tool"]["flwr"]["app"].get("config", {})
|
|
116
|
+
flat_config_flat = flatten_dict(default_config)
|
|
117
|
+
return fuse_dicts(flat_config_flat, run.override_config)
|
|
100
118
|
|
|
101
119
|
|
|
102
120
|
def get_fused_config(run: Run, flwr_dir: Optional[Path]) -> UserConfig:
|
|
@@ -160,7 +178,6 @@ def unflatten_dict(flat_dict: Dict[str, Any]) -> Dict[str, Any]:
|
|
|
160
178
|
|
|
161
179
|
def parse_config_args(
|
|
162
180
|
config: Optional[List[str]],
|
|
163
|
-
separator: str = ",",
|
|
164
181
|
) -> UserConfig:
|
|
165
182
|
"""Parse separator separated list of key-value pairs separated by '='."""
|
|
166
183
|
overrides: UserConfig = {}
|
|
@@ -168,18 +185,22 @@ def parse_config_args(
|
|
|
168
185
|
if config is None:
|
|
169
186
|
return overrides
|
|
170
187
|
|
|
188
|
+
# Regular expression to capture key-value pairs with possible quoted values
|
|
189
|
+
pattern = re.compile(r"(\S+?)=(\'[^\']*\'|\"[^\"]*\"|\S+)")
|
|
190
|
+
|
|
171
191
|
for config_line in config:
|
|
172
192
|
if config_line:
|
|
173
|
-
|
|
193
|
+
matches = pattern.findall(config_line)
|
|
194
|
+
|
|
174
195
|
if (
|
|
175
|
-
len(
|
|
176
|
-
and "=" not in
|
|
177
|
-
and
|
|
196
|
+
len(matches) == 1
|
|
197
|
+
and "=" not in matches[0][0]
|
|
198
|
+
and matches[0][0].endswith(".toml")
|
|
178
199
|
):
|
|
179
|
-
with Path(
|
|
200
|
+
with Path(matches[0][0]).open("rb") as config_file:
|
|
180
201
|
overrides = flatten_dict(tomli.load(config_file))
|
|
181
202
|
else:
|
|
182
|
-
toml_str = "\n".join(
|
|
203
|
+
toml_str = "\n".join(f"{k} = {v}" for k, v in matches)
|
|
183
204
|
overrides.update(tomli.loads(toml_str))
|
|
184
205
|
|
|
185
206
|
return overrides
|
|
@@ -58,27 +58,61 @@ def _check_value(value: ConfigsRecordValues) -> None:
|
|
|
58
58
|
|
|
59
59
|
|
|
60
60
|
class ConfigsRecord(TypedDict[str, ConfigsRecordValues]):
|
|
61
|
-
"""Configs record.
|
|
61
|
+
"""Configs record.
|
|
62
|
+
|
|
63
|
+
A :code:`ConfigsRecord` is a Python dictionary designed to ensure that
|
|
64
|
+
each key-value pair adheres to specified data types. A :code:`ConfigsRecord`
|
|
65
|
+
is one of the types of records that a
|
|
66
|
+
`flwr.common.RecordSet <flwr.common.RecordSet.html#recordset>`_ supports and
|
|
67
|
+
can therefore be used to construct :code:`common.Message` objects.
|
|
68
|
+
|
|
69
|
+
Parameters
|
|
70
|
+
----------
|
|
71
|
+
configs_dict : Optional[Dict[str, ConfigsRecordValues]]
|
|
72
|
+
A dictionary that stores basic types (i.e. `str`, `int`, `float`, `bytes` as
|
|
73
|
+
defined in `ConfigsScalar`) and lists of such types (see
|
|
74
|
+
`ConfigsScalarList`).
|
|
75
|
+
keep_input : bool (default: True)
|
|
76
|
+
A boolean indicating whether config passed should be deleted from the input
|
|
77
|
+
dictionary immediately after adding them to the record. When set
|
|
78
|
+
to True, the data is duplicated in memory. If memory is a concern, set
|
|
79
|
+
it to False.
|
|
80
|
+
|
|
81
|
+
Examples
|
|
82
|
+
--------
|
|
83
|
+
The usage of a :code:`ConfigsRecord` is envisioned for sending configuration values
|
|
84
|
+
telling the target node how to perform a certain action (e.g. train/evaluate a model
|
|
85
|
+
). You can use standard Python built-in types such as :code:`float`, :code:`str`
|
|
86
|
+
, :code:`bytes`. All types allowed are defined in
|
|
87
|
+
:code:`flwr.common.ConfigsRecordValues`. While lists are supported, we
|
|
88
|
+
encourage you to use a :code:`ParametersRecord` instead if these are of high
|
|
89
|
+
dimensionality.
|
|
90
|
+
|
|
91
|
+
Let's see some examples of how to construct a :code:`ConfigsRecord` from scratch:
|
|
92
|
+
|
|
93
|
+
>>> from flwr.common import ConfigsRecord
|
|
94
|
+
>>>
|
|
95
|
+
>>> # A `ConfigsRecord` is a specialized Python dictionary
|
|
96
|
+
>>> record = ConfigsRecord({"lr": 0.1, "batch-size": 128})
|
|
97
|
+
>>> # You can add more content to an existing record
|
|
98
|
+
>>> record["compute-average"] = True
|
|
99
|
+
>>> # It also supports lists
|
|
100
|
+
>>> record["loss-fn-coefficients"] = [0.4, 0.25, 0.35]
|
|
101
|
+
>>> # And string values (among other types)
|
|
102
|
+
>>> record["path-to-S3"] = "s3://bucket_name/folder1/fileA.json"
|
|
103
|
+
|
|
104
|
+
Just like the other types of records in a :code:`flwr.common.RecordSet`, types are
|
|
105
|
+
enforced. If you need to add a custom data structure or object, we recommend to
|
|
106
|
+
serialise it into bytes and save it as such (bytes are allowed in a
|
|
107
|
+
:code:`ConfigsRecord`)
|
|
108
|
+
"""
|
|
62
109
|
|
|
63
110
|
def __init__(
|
|
64
111
|
self,
|
|
65
112
|
configs_dict: Optional[Dict[str, ConfigsRecordValues]] = None,
|
|
66
113
|
keep_input: bool = True,
|
|
67
114
|
) -> None:
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
Parameters
|
|
71
|
-
----------
|
|
72
|
-
configs_dict : Optional[Dict[str, ConfigsRecordValues]]
|
|
73
|
-
A dictionary that stores basic types (i.e. `str`, `int`, `float`, `bytes` as
|
|
74
|
-
defined in `ConfigsScalar`) and lists of such types (see
|
|
75
|
-
`ConfigsScalarList`).
|
|
76
|
-
keep_input : bool (default: True)
|
|
77
|
-
A boolean indicating whether config passed should be deleted from the input
|
|
78
|
-
dictionary immediately after adding them to the record. When set
|
|
79
|
-
to True, the data is duplicated in memory. If memory is a concern, set
|
|
80
|
-
it to False.
|
|
81
|
-
"""
|
|
115
|
+
|
|
82
116
|
super().__init__(_check_key, _check_value)
|
|
83
117
|
if configs_dict:
|
|
84
118
|
for k in list(configs_dict.keys()):
|
|
@@ -58,26 +58,66 @@ def _check_value(value: MetricsRecordValues) -> None:
|
|
|
58
58
|
|
|
59
59
|
|
|
60
60
|
class MetricsRecord(TypedDict[str, MetricsRecordValues]):
|
|
61
|
-
"""Metrics
|
|
61
|
+
"""Metrics recod.
|
|
62
|
+
|
|
63
|
+
A :code:`MetricsRecord` is a Python dictionary designed to ensure that
|
|
64
|
+
each key-value pair adheres to specified data types. A :code:`MetricsRecord`
|
|
65
|
+
is one of the types of records that a
|
|
66
|
+
`flwr.common.RecordSet <flwr.common.RecordSet.html#recordset>`_ supports and
|
|
67
|
+
can therefore be used to construct :code:`common.Message` objects.
|
|
68
|
+
|
|
69
|
+
Parameters
|
|
70
|
+
----------
|
|
71
|
+
metrics_dict : Optional[Dict[str, MetricsRecordValues]]
|
|
72
|
+
A dictionary that stores basic types (i.e. `int`, `float` as defined
|
|
73
|
+
in `MetricsScalar`) and list of such types (see `MetricsScalarList`).
|
|
74
|
+
keep_input : bool (default: True)
|
|
75
|
+
A boolean indicating whether metrics should be deleted from the input
|
|
76
|
+
dictionary immediately after adding them to the record. When set
|
|
77
|
+
to True, the data is duplicated in memory. If memory is a concern, set
|
|
78
|
+
it to False.
|
|
79
|
+
|
|
80
|
+
Examples
|
|
81
|
+
--------
|
|
82
|
+
The usage of a :code:`MetricsRecord` is envisioned for communicating results
|
|
83
|
+
obtained when a node performs an action. A few typical examples include:
|
|
84
|
+
communicating the training accuracy after a model is trained locally by a
|
|
85
|
+
:code:`ClientApp`, reporting the validation loss obtained at a :code:`ClientApp`,
|
|
86
|
+
or, more generally, the output of executing a query by the :code:`ClientApp`.
|
|
87
|
+
Common to these examples is that the output can be typically represented by
|
|
88
|
+
a single scalar (:code:`int`, :code:`float`) or list of scalars.
|
|
89
|
+
|
|
90
|
+
Let's see some examples of how to construct a :code:`MetricsRecord` from scratch:
|
|
91
|
+
|
|
92
|
+
>>> from flwr.common import MetricsRecord
|
|
93
|
+
>>>
|
|
94
|
+
>>> # A `MetricsRecord` is a specialized Python dictionary
|
|
95
|
+
>>> record = MetricsRecord({"accuracy": 0.94})
|
|
96
|
+
>>> # You can add more content to an existing record
|
|
97
|
+
>>> record["loss"] = 0.01
|
|
98
|
+
>>> # It also supports lists
|
|
99
|
+
>>> record["loss-historic"] = [0.9, 0.5, 0.01]
|
|
100
|
+
|
|
101
|
+
Since types are enforced, the types of the objects inserted are checked. For a
|
|
102
|
+
:code:`MetricsRecord`, value types allowed are those in defined in
|
|
103
|
+
:code:`flwr.common.MetricsRecordValues`. Similarly, only :code:`str` keys are
|
|
104
|
+
allowed.
|
|
105
|
+
|
|
106
|
+
>>> from flwr.common import MetricsRecord
|
|
107
|
+
>>>
|
|
108
|
+
>>> record = MetricsRecord() # an empty record
|
|
109
|
+
>>> # Add unsupported value
|
|
110
|
+
>>> record["something-unsupported"] = {'a': 123} # Will throw a `TypeError`
|
|
111
|
+
|
|
112
|
+
If you need a more versatily type of record try :code:`ConfigsRecord` or
|
|
113
|
+
:code:`ParametersRecord`.
|
|
114
|
+
"""
|
|
62
115
|
|
|
63
116
|
def __init__(
|
|
64
117
|
self,
|
|
65
118
|
metrics_dict: Optional[Dict[str, MetricsRecordValues]] = None,
|
|
66
119
|
keep_input: bool = True,
|
|
67
120
|
):
|
|
68
|
-
"""Construct a MetricsRecord object.
|
|
69
|
-
|
|
70
|
-
Parameters
|
|
71
|
-
----------
|
|
72
|
-
metrics_dict : Optional[Dict[str, MetricsRecordValues]]
|
|
73
|
-
A dictionary that stores basic types (i.e. `int`, `float` as defined
|
|
74
|
-
in `MetricsScalar`) and list of such types (see `MetricsScalarList`).
|
|
75
|
-
keep_input : bool (default: True)
|
|
76
|
-
A boolean indicating whether metrics should be deleted from the input
|
|
77
|
-
dictionary immediately after adding them to the record. When set
|
|
78
|
-
to True, the data is duplicated in memory. If memory is a concern, set
|
|
79
|
-
it to False.
|
|
80
|
-
"""
|
|
81
121
|
super().__init__(_check_key, _check_value)
|
|
82
122
|
if metrics_dict:
|
|
83
123
|
for k in list(metrics_dict.keys()):
|
|
@@ -83,11 +83,93 @@ def _check_value(value: Array) -> None:
|
|
|
83
83
|
|
|
84
84
|
|
|
85
85
|
class ParametersRecord(TypedDict[str, Array]):
|
|
86
|
-
"""Parameters record.
|
|
86
|
+
r"""Parameters record.
|
|
87
87
|
|
|
88
88
|
A dataclass storing named Arrays in order. This means that it holds entries as an
|
|
89
89
|
OrderedDict[str, Array]. ParametersRecord objects can be viewed as an equivalent to
|
|
90
|
-
PyTorch's state_dict, but holding serialised tensors instead.
|
|
90
|
+
PyTorch's state_dict, but holding serialised tensors instead. A
|
|
91
|
+
:code:`ParametersRecord` is one of the types of records that a
|
|
92
|
+
`flwr.common.RecordSet <flwr.common.RecordSet.html#recordset>`_ supports and
|
|
93
|
+
can therefore be used to construct :code:`common.Message` objects.
|
|
94
|
+
|
|
95
|
+
Parameters
|
|
96
|
+
----------
|
|
97
|
+
array_dict : Optional[OrderedDict[str, Array]]
|
|
98
|
+
A dictionary that stores serialized array-like or tensor-like objects.
|
|
99
|
+
keep_input : bool (default: False)
|
|
100
|
+
A boolean indicating whether parameters should be deleted from the input
|
|
101
|
+
dictionary immediately after adding them to the record. If False, the
|
|
102
|
+
dictionary passed to `set_parameters()` will be empty once exiting from that
|
|
103
|
+
function. This is the desired behaviour when working with very large
|
|
104
|
+
models/tensors/arrays. However, if you plan to continue working with your
|
|
105
|
+
parameters after adding it to the record, set this flag to True. When set
|
|
106
|
+
to True, the data is duplicated in memory.
|
|
107
|
+
|
|
108
|
+
Examples
|
|
109
|
+
--------
|
|
110
|
+
The usage of :code:`ParametersRecord` is envisioned for storing data arrays (e.g.
|
|
111
|
+
parameters of a machine learning model). These first need to be serialized into
|
|
112
|
+
a :code:`flwr.common.Array` data structure.
|
|
113
|
+
|
|
114
|
+
Let's see some examples:
|
|
115
|
+
|
|
116
|
+
>>> import numpy as np
|
|
117
|
+
>>> from flwr.common import ParametersRecord
|
|
118
|
+
>>> from flwr.common import array_from_numpy
|
|
119
|
+
>>>
|
|
120
|
+
>>> # Let's create a simple NumPy array
|
|
121
|
+
>>> arr_np = np.random.randn(3, 3)
|
|
122
|
+
>>>
|
|
123
|
+
>>> # If we print it
|
|
124
|
+
>>> array([[-1.84242409, -1.01539537, -0.46528405],
|
|
125
|
+
>>> [ 0.32991896, 0.55540414, 0.44085534],
|
|
126
|
+
>>> [-0.10758364, 1.97619858, -0.37120501]])
|
|
127
|
+
>>>
|
|
128
|
+
>>> # Let's create an Array out of it
|
|
129
|
+
>>> arr = array_from_numpy(arr_np)
|
|
130
|
+
>>>
|
|
131
|
+
>>> # If we print it you'll see (note the binary data)
|
|
132
|
+
>>> Array(dtype='float64', shape=[3,3], stype='numpy.ndarray', data=b'@\x99\x18...')
|
|
133
|
+
>>>
|
|
134
|
+
>>> # Adding it to a ParametersRecord:
|
|
135
|
+
>>> p_record = ParametersRecord({"my_array": arr})
|
|
136
|
+
|
|
137
|
+
Now that the NumPy array is embedded into a :code:`ParametersRecord` it could be
|
|
138
|
+
sent if added as part of a :code:`common.Message` or it could be saved as a
|
|
139
|
+
persistent state of a :code:`ClientApp` via its context. Regardless of the usecase,
|
|
140
|
+
we will sooner or later want to recover the array in its original NumPy
|
|
141
|
+
representation. For the example above, where the array was serialized using the
|
|
142
|
+
built-in utility function, deserialization can be done as follows:
|
|
143
|
+
|
|
144
|
+
>>> # Use the Array's built-in method
|
|
145
|
+
>>> arr_np_d = arr.numpy()
|
|
146
|
+
>>>
|
|
147
|
+
>>> # If printed, it will show the exact same data as above:
|
|
148
|
+
>>> array([[-1.84242409, -1.01539537, -0.46528405],
|
|
149
|
+
>>> [ 0.32991896, 0.55540414, 0.44085534],
|
|
150
|
+
>>> [-0.10758364, 1.97619858, -0.37120501]])
|
|
151
|
+
|
|
152
|
+
If you need finer control on how your arrays are serialized and deserialized, you
|
|
153
|
+
can construct :code:`Array` objects directly like this:
|
|
154
|
+
|
|
155
|
+
>>> from flwr.common import Array
|
|
156
|
+
>>> # Serialize your array and construct Array object
|
|
157
|
+
>>> arr = Array(
|
|
158
|
+
>>> data=ndarray.tobytes(),
|
|
159
|
+
>>> dtype=str(ndarray.dtype),
|
|
160
|
+
>>> stype="", # Could be used in a deserialization function
|
|
161
|
+
>>> shape=list(ndarray.shape),
|
|
162
|
+
>>> )
|
|
163
|
+
>>>
|
|
164
|
+
>>> # Then you can deserialize it like this
|
|
165
|
+
>>> arr_np_d = np.frombuffer(
|
|
166
|
+
>>> buffer=array.data,
|
|
167
|
+
>>> dtype=array.dtype,
|
|
168
|
+
>>> ).reshape(array.shape)
|
|
169
|
+
|
|
170
|
+
Note that different arrays (e.g. from PyTorch, Tensorflow) might require different
|
|
171
|
+
serialization mechanism. Howerver, they often support a conversion to NumPy,
|
|
172
|
+
therefore allowing to use the same or similar steps as in the example above.
|
|
91
173
|
"""
|
|
92
174
|
|
|
93
175
|
def __init__(
|
|
@@ -95,21 +177,6 @@ class ParametersRecord(TypedDict[str, Array]):
|
|
|
95
177
|
array_dict: Optional[OrderedDict[str, Array]] = None,
|
|
96
178
|
keep_input: bool = False,
|
|
97
179
|
) -> None:
|
|
98
|
-
"""Construct a ParametersRecord object.
|
|
99
|
-
|
|
100
|
-
Parameters
|
|
101
|
-
----------
|
|
102
|
-
array_dict : Optional[OrderedDict[str, Array]]
|
|
103
|
-
A dictionary that stores serialized array-like or tensor-like objects.
|
|
104
|
-
keep_input : bool (default: False)
|
|
105
|
-
A boolean indicating whether parameters should be deleted from the input
|
|
106
|
-
dictionary immediately after adding them to the record. If False, the
|
|
107
|
-
dictionary passed to `set_parameters()` will be empty once exiting from that
|
|
108
|
-
function. This is the desired behaviour when working with very large
|
|
109
|
-
models/tensors/arrays. However, if you plan to continue working with your
|
|
110
|
-
parameters after adding it to the record, set this flag to True. When set
|
|
111
|
-
to True, the data is duplicated in memory.
|
|
112
|
-
"""
|
|
113
180
|
super().__init__(_check_key, _check_value)
|
|
114
181
|
if array_dict:
|
|
115
182
|
for k in list(array_dict.keys()):
|