flwr-nightly 1.21.0.dev20250901__py3-none-any.whl → 1.21.0.dev20250903__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flwr/cli/constant.py +25 -8
- flwr/cli/new/templates/app/code/server.pytorch_msg_api.py.tpl +0 -8
- flwr/cli/run/run.py +2 -6
- flwr/clientapp/__init__.py +4 -0
- flwr/clientapp/centraldp_mods.py +132 -0
- flwr/common/exception.py +31 -0
- flwr/common/exit/__init__.py +4 -0
- flwr/common/exit/exit.py +4 -0
- flwr/common/exit/exit_code.py +7 -0
- flwr/common/exit/exit_handler.py +62 -0
- flwr/common/{exit_handlers.py → exit/signal_handler.py} +20 -37
- flwr/common/grpc.py +0 -11
- flwr/common/inflatable_utils.py +1 -1
- flwr/compat/server/app.py +2 -2
- flwr/server/app.py +12 -3
- flwr/server/serverapp/app.py +41 -28
- flwr/serverapp/dp_fixed_clipping.py +352 -0
- flwr/serverapp/strategy/__init__.py +6 -0
- flwr/serverapp/strategy/dp_fixed_clipping.py +352 -0
- flwr/serverapp/strategy/result.py +76 -2
- flwr/serverapp/strategy/strategy.py +15 -20
- flwr/serverapp/strategy/strategy_utils.py +5 -1
- flwr/supercore/cli/flower_superexec.py +3 -0
- flwr/supercore/grpc_health/__init__.py +3 -0
- flwr/supercore/grpc_health/health_server.py +53 -0
- flwr/supercore/grpc_health/simple_health_servicer.py +2 -2
- flwr/supercore/superexec/run_superexec.py +15 -3
- flwr/supernode/cli/flower_supernode.py +3 -0
- flwr/supernode/start_client_internal.py +15 -4
- {flwr_nightly-1.21.0.dev20250901.dist-info → flwr_nightly-1.21.0.dev20250903.dist-info}/METADATA +1 -1
- {flwr_nightly-1.21.0.dev20250901.dist-info → flwr_nightly-1.21.0.dev20250903.dist-info}/RECORD +33 -27
- {flwr_nightly-1.21.0.dev20250901.dist-info → flwr_nightly-1.21.0.dev20250903.dist-info}/WHEEL +0 -0
- {flwr_nightly-1.21.0.dev20250901.dist-info → flwr_nightly-1.21.0.dev20250903.dist-info}/entry_points.txt +0 -0
flwr/cli/constant.py
CHANGED
@@ -15,13 +15,30 @@
|
|
15
15
|
"""Constants for CLI commands."""
|
16
16
|
|
17
17
|
|
18
|
+
# General help message for config overrides
|
19
|
+
CONFIG_HELP_MESSAGE = (
|
20
|
+
"Override {0} values using one of the following formats:\n\n"
|
21
|
+
"--{1} '<k1>=<v1> <k2>=<v2>' | --{1} '<k1>=<v1>' --{1} '<k2>=<v2>'{2}\n\n"
|
22
|
+
"When providing key-value pairs, values can be of any type supported by TOML "
|
23
|
+
"(e.g., bool, int, float, string). The specified keys (<k1> and <k2> in the "
|
24
|
+
"example) must exist in the {0} under the `{3}` section of `pyproject.toml` to be "
|
25
|
+
"overridden.{4}"
|
26
|
+
)
|
27
|
+
|
28
|
+
# The help message for `--run-config` option
|
29
|
+
RUN_CONFIG_HELP_MESSAGE = CONFIG_HELP_MESSAGE.format(
|
30
|
+
"run configuration",
|
31
|
+
"run-config",
|
32
|
+
" | --run-config <path/to/your/toml>",
|
33
|
+
"[tool.flwr.app.config]",
|
34
|
+
" Alternatively, provide a TOML file containing overrides.",
|
35
|
+
)
|
36
|
+
|
18
37
|
# The help message for `--federation-config` option
|
19
|
-
FEDERATION_CONFIG_HELP_MESSAGE = (
|
20
|
-
"
|
21
|
-
"
|
22
|
-
"
|
23
|
-
"
|
24
|
-
"
|
25
|
-
"`[tool.flwr.federations.<YOUR_FEDERATION>]` table of the `pyproject.toml` "
|
26
|
-
"for proper overriding."
|
38
|
+
FEDERATION_CONFIG_HELP_MESSAGE = CONFIG_HELP_MESSAGE.format(
|
39
|
+
"federation configuration",
|
40
|
+
"federation-config",
|
41
|
+
"",
|
42
|
+
"[tool.flwr.federations.<YOUR-FEDERATION>]",
|
43
|
+
"",
|
27
44
|
)
|
@@ -1,7 +1,5 @@
|
|
1
1
|
"""$project_name: A Flower / $framework_str app."""
|
2
2
|
|
3
|
-
from pprint import pprint
|
4
|
-
|
5
3
|
import torch
|
6
4
|
from flwr.common import ArrayRecord, ConfigRecord, Context
|
7
5
|
from flwr.server import Grid, ServerApp
|
@@ -37,12 +35,6 @@ def main(grid: Grid, context: Context) -> None:
|
|
37
35
|
num_rounds=num_rounds,
|
38
36
|
)
|
39
37
|
|
40
|
-
# Log resulting metrics
|
41
|
-
print("\nDistributed train metrics:")
|
42
|
-
pprint(result.train_metrics_clientapp)
|
43
|
-
print("\nDistributed evaluate metrics:")
|
44
|
-
pprint(result.evaluate_metrics_clientapp)
|
45
|
-
|
46
38
|
# Save final model to disk
|
47
39
|
print("\nSaving final model to disk...")
|
48
40
|
state_dict = result.arrays.to_torch_state_dict()
|
flwr/cli/run/run.py
CHANGED
@@ -30,7 +30,7 @@ from flwr.cli.config_utils import (
|
|
30
30
|
process_loaded_project_config,
|
31
31
|
validate_federation_in_project_config,
|
32
32
|
)
|
33
|
-
from flwr.cli.constant import FEDERATION_CONFIG_HELP_MESSAGE
|
33
|
+
from flwr.cli.constant import FEDERATION_CONFIG_HELP_MESSAGE, RUN_CONFIG_HELP_MESSAGE
|
34
34
|
from flwr.common.config import (
|
35
35
|
flatten_dict,
|
36
36
|
get_metadata_from_config,
|
@@ -65,11 +65,7 @@ def run(
|
|
65
65
|
typer.Option(
|
66
66
|
"--run-config",
|
67
67
|
"-c",
|
68
|
-
help=
|
69
|
-
"`--run-config 'key1=value1 key2=value2' --run-config 'key3=value3'`\n\n"
|
70
|
-
"Values can be of any type supported in TOML, such as bool, int, "
|
71
|
-
"float, or string. Ensure that the keys (`key1`, `key2`, `key3` "
|
72
|
-
"in this example) exist in `pyproject.toml` for proper overriding.",
|
68
|
+
help=RUN_CONFIG_HELP_MESSAGE,
|
73
69
|
),
|
74
70
|
] = None,
|
75
71
|
federation_config_overrides: Annotated[
|
flwr/clientapp/__init__.py
CHANGED
@@ -0,0 +1,132 @@
|
|
1
|
+
# Copyright 2025 Flower Labs GmbH. All Rights Reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
# ==============================================================================
|
15
|
+
"""Clipping modifiers for central DP with client-side clipping."""
|
16
|
+
|
17
|
+
|
18
|
+
from collections import OrderedDict
|
19
|
+
from logging import INFO, WARN
|
20
|
+
from typing import cast
|
21
|
+
|
22
|
+
from flwr.client.typing import ClientAppCallable
|
23
|
+
from flwr.common import Array, ArrayRecord, Context, Message, MessageType, log
|
24
|
+
from flwr.common.differential_privacy import compute_clip_model_update
|
25
|
+
from flwr.common.differential_privacy_constants import KEY_CLIPPING_NORM
|
26
|
+
|
27
|
+
|
28
|
+
# pylint: disable=too-many-return-statements
|
29
|
+
def fixedclipping_mod(
|
30
|
+
msg: Message, ctxt: Context, call_next: ClientAppCallable
|
31
|
+
) -> Message:
|
32
|
+
"""Client-side fixed clipping modifier.
|
33
|
+
|
34
|
+
This mod needs to be used with the `DifferentialPrivacyClientSideFixedClipping`
|
35
|
+
server-side strategy wrapper.
|
36
|
+
|
37
|
+
The wrapper sends the clipping_norm value to the client.
|
38
|
+
|
39
|
+
This mod clips the client model updates before sending them to the server.
|
40
|
+
|
41
|
+
It operates on messages of type `MessageType.TRAIN`.
|
42
|
+
|
43
|
+
Notes
|
44
|
+
-----
|
45
|
+
Consider the order of mods when using multiple.
|
46
|
+
|
47
|
+
Typically, fixedclipping_mod should be the last to operate on params.
|
48
|
+
"""
|
49
|
+
if msg.metadata.message_type != MessageType.TRAIN:
|
50
|
+
return call_next(msg, ctxt)
|
51
|
+
|
52
|
+
if len(msg.content.array_records) != 1:
|
53
|
+
log(
|
54
|
+
WARN,
|
55
|
+
"fixedclipping_mod is designed to work with a single ArrayRecord. "
|
56
|
+
"Skipping.",
|
57
|
+
)
|
58
|
+
return call_next(msg, ctxt)
|
59
|
+
|
60
|
+
if len(msg.content.config_records) != 1:
|
61
|
+
log(
|
62
|
+
WARN,
|
63
|
+
"fixedclipping_mod is designed to work with a single ConfigRecord. "
|
64
|
+
"Skipping.",
|
65
|
+
)
|
66
|
+
return call_next(msg, ctxt)
|
67
|
+
|
68
|
+
# Get keys in the single ConfigRecord
|
69
|
+
keys_in_config = set(next(iter(msg.content.config_records.values())).keys())
|
70
|
+
if KEY_CLIPPING_NORM not in keys_in_config:
|
71
|
+
raise KeyError(
|
72
|
+
f"The {KEY_CLIPPING_NORM} value is not supplied by the "
|
73
|
+
f"`DifferentialPrivacyClientSideFixedClipping` wrapper at"
|
74
|
+
f" the server side."
|
75
|
+
)
|
76
|
+
# Record array record communicated to client and clipping norm
|
77
|
+
original_array_record = next(iter(msg.content.array_records.values()))
|
78
|
+
clipping_norm = cast(
|
79
|
+
float, next(iter(msg.content.config_records.values()))[KEY_CLIPPING_NORM]
|
80
|
+
)
|
81
|
+
|
82
|
+
# Call inner app
|
83
|
+
out_msg = call_next(msg, ctxt)
|
84
|
+
|
85
|
+
# Check if the msg has error
|
86
|
+
if out_msg.has_error():
|
87
|
+
return out_msg
|
88
|
+
|
89
|
+
# Ensure there is a single ArrayRecord
|
90
|
+
if len(out_msg.content.array_records) != 1:
|
91
|
+
log(
|
92
|
+
WARN,
|
93
|
+
"fixedclipping_mod is designed to work with a single ArrayRecord. "
|
94
|
+
"Skipping.",
|
95
|
+
)
|
96
|
+
return out_msg
|
97
|
+
|
98
|
+
new_array_record_key, client_to_server_arrecord = next(
|
99
|
+
iter(out_msg.content.array_records.items())
|
100
|
+
)
|
101
|
+
# Ensure keys in returned ArrayRecord match those in the one sent from server
|
102
|
+
if set(original_array_record.keys()) != set(client_to_server_arrecord.keys()):
|
103
|
+
log(
|
104
|
+
WARN,
|
105
|
+
"fixedclipping_mod: Keys in ArrayRecord must match those from the model "
|
106
|
+
"that the ClientApp received. Skipping.",
|
107
|
+
)
|
108
|
+
return out_msg
|
109
|
+
|
110
|
+
client_to_server_ndarrays = client_to_server_arrecord.to_numpy_ndarrays()
|
111
|
+
# Clip the client update
|
112
|
+
compute_clip_model_update(
|
113
|
+
param1=client_to_server_ndarrays,
|
114
|
+
param2=original_array_record.to_numpy_ndarrays(),
|
115
|
+
clipping_norm=clipping_norm,
|
116
|
+
)
|
117
|
+
|
118
|
+
log(
|
119
|
+
INFO, "fixedclipping_mod: parameters are clipped by value: %.4f.", clipping_norm
|
120
|
+
)
|
121
|
+
# Replace outgoing ArrayRecord's Array while preserving their keys
|
122
|
+
out_msg.content.array_records[new_array_record_key] = ArrayRecord(
|
123
|
+
OrderedDict(
|
124
|
+
{
|
125
|
+
k: Array(v)
|
126
|
+
for k, v in zip(
|
127
|
+
client_to_server_arrecord.keys(), client_to_server_ndarrays
|
128
|
+
)
|
129
|
+
}
|
130
|
+
)
|
131
|
+
)
|
132
|
+
return out_msg
|
flwr/common/exception.py
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
# Copyright 2025 Flower Labs GmbH. All Rights Reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
# ==============================================================================
|
15
|
+
"""Flower application exceptions."""
|
16
|
+
|
17
|
+
|
18
|
+
class AppExitException(Exception):
|
19
|
+
"""Base exception for all application-level errors in ServerApp and ClientApp.
|
20
|
+
|
21
|
+
When raised, the process will exit and report a telemetry event with the associated
|
22
|
+
exit code.
|
23
|
+
"""
|
24
|
+
|
25
|
+
# Default exit code — subclasses must override
|
26
|
+
exit_code = -1
|
27
|
+
|
28
|
+
def __init_subclass__(cls) -> None:
|
29
|
+
"""Ensure subclasses override the exit_code attribute."""
|
30
|
+
if cls.exit_code == -1:
|
31
|
+
raise ValueError("Subclasses must override the exit_code attribute.")
|
flwr/common/exit/__init__.py
CHANGED
@@ -17,8 +17,12 @@
|
|
17
17
|
|
18
18
|
from .exit import flwr_exit
|
19
19
|
from .exit_code import ExitCode
|
20
|
+
from .exit_handler import add_exit_handler
|
21
|
+
from .signal_handler import register_signal_handlers
|
20
22
|
|
21
23
|
__all__ = [
|
22
24
|
"ExitCode",
|
25
|
+
"add_exit_handler",
|
23
26
|
"flwr_exit",
|
27
|
+
"register_signal_handlers",
|
24
28
|
]
|
flwr/common/exit/exit.py
CHANGED
@@ -26,6 +26,7 @@ from flwr.common.version import package_version
|
|
26
26
|
|
27
27
|
from ..logger import log
|
28
28
|
from .exit_code import EXIT_CODE_HELP
|
29
|
+
from .exit_handler import trigger_exit_handlers
|
29
30
|
|
30
31
|
HELP_PAGE_URL = (
|
31
32
|
f"https://flower.ai/docs/framework/v{package_version}/en/ref-exit-codes/"
|
@@ -80,6 +81,9 @@ def flwr_exit(
|
|
80
81
|
# Log the exit message
|
81
82
|
log(log_level, exit_message)
|
82
83
|
|
84
|
+
# Trigger exit handlers
|
85
|
+
trigger_exit_handlers()
|
86
|
+
|
83
87
|
# Exit
|
84
88
|
sys.exit(sys_exit_code)
|
85
89
|
|
flwr/common/exit/exit_code.py
CHANGED
@@ -36,6 +36,8 @@ class ExitCode:
|
|
36
36
|
|
37
37
|
# ServerApp-specific exit codes (200-299)
|
38
38
|
SERVERAPP_STRATEGY_PRECONDITION_UNMET = 200
|
39
|
+
SERVERAPP_EXCEPTION = 201
|
40
|
+
SERVERAPP_STRATEGY_AGGREGATION_ERROR = 202
|
39
41
|
|
40
42
|
# SuperNode-specific exit codes (300-399)
|
41
43
|
SUPERNODE_REST_ADDRESS_INVALID = 300
|
@@ -89,6 +91,11 @@ EXIT_CODE_HELP = {
|
|
89
91
|
"perform weighted average (e.g. in FedAvg) please ensure the returned "
|
90
92
|
"MetricRecord from ClientApps do include this key."
|
91
93
|
),
|
94
|
+
ExitCode.SERVERAPP_EXCEPTION: "An unhandled exception occurred in the ServerApp.",
|
95
|
+
ExitCode.SERVERAPP_STRATEGY_AGGREGATION_ERROR: (
|
96
|
+
"The strategy encountered an error during aggregation. Please check the logs "
|
97
|
+
"for more details."
|
98
|
+
),
|
92
99
|
# SuperNode-specific exit codes (300-399)
|
93
100
|
ExitCode.SUPERNODE_REST_ADDRESS_INVALID: (
|
94
101
|
"When using the REST API, please provide `https://` or "
|
@@ -0,0 +1,62 @@
|
|
1
|
+
# Copyright 2025 Flower Labs GmbH. All Rights Reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
# ==============================================================================
|
15
|
+
"""Common function to register exit handlers."""
|
16
|
+
|
17
|
+
|
18
|
+
import signal
|
19
|
+
import threading
|
20
|
+
from typing import Callable
|
21
|
+
|
22
|
+
from .exit_code import ExitCode
|
23
|
+
|
24
|
+
SIGNAL_TO_EXIT_CODE: dict[int, int] = {
|
25
|
+
signal.SIGINT: ExitCode.GRACEFUL_EXIT_SIGINT,
|
26
|
+
signal.SIGTERM: ExitCode.GRACEFUL_EXIT_SIGTERM,
|
27
|
+
}
|
28
|
+
registered_exit_handlers: list[Callable[[], None]] = []
|
29
|
+
_lock_handlers = threading.Lock()
|
30
|
+
|
31
|
+
# SIGQUIT is not available on Windows
|
32
|
+
if hasattr(signal, "SIGQUIT"):
|
33
|
+
SIGNAL_TO_EXIT_CODE[signal.SIGQUIT] = ExitCode.GRACEFUL_EXIT_SIGQUIT
|
34
|
+
|
35
|
+
|
36
|
+
def add_exit_handler(exit_handler: Callable[[], None]) -> None:
|
37
|
+
"""Add an exit handler to be called on graceful exit.
|
38
|
+
|
39
|
+
This function allows you to register additional exit handlers
|
40
|
+
that will be executed when `flwr_exit` is called.
|
41
|
+
|
42
|
+
Parameters
|
43
|
+
----------
|
44
|
+
exit_handler : Callable[[], None]
|
45
|
+
A callable that takes no arguments and performs cleanup or
|
46
|
+
other actions before the application exits.
|
47
|
+
|
48
|
+
Notes
|
49
|
+
-----
|
50
|
+
The registered exit handlers will be called in LIFO order, i.e.,
|
51
|
+
the last registered handler will be the first to be called.
|
52
|
+
"""
|
53
|
+
with _lock_handlers:
|
54
|
+
registered_exit_handlers.append(exit_handler)
|
55
|
+
|
56
|
+
|
57
|
+
def trigger_exit_handlers() -> None:
|
58
|
+
"""Trigger all registered exit handlers in LIFO order."""
|
59
|
+
with _lock_handlers:
|
60
|
+
for handler in reversed(registered_exit_handlers):
|
61
|
+
handler()
|
62
|
+
registered_exit_handlers.clear()
|
@@ -12,7 +12,7 @@
|
|
12
12
|
# See the License for the specific language governing permissions and
|
13
13
|
# limitations under the License.
|
14
14
|
# ==============================================================================
|
15
|
-
"""Common function to register
|
15
|
+
"""Common function to register signal handlers."""
|
16
16
|
|
17
17
|
|
18
18
|
import signal
|
@@ -24,20 +24,21 @@ from grpc import Server
|
|
24
24
|
|
25
25
|
from flwr.common.telemetry import EventType
|
26
26
|
|
27
|
-
from .exit import
|
27
|
+
from .exit import flwr_exit
|
28
|
+
from .exit_code import ExitCode
|
29
|
+
from .exit_handler import add_exit_handler
|
28
30
|
|
29
31
|
SIGNAL_TO_EXIT_CODE: dict[int, int] = {
|
30
32
|
signal.SIGINT: ExitCode.GRACEFUL_EXIT_SIGINT,
|
31
33
|
signal.SIGTERM: ExitCode.GRACEFUL_EXIT_SIGTERM,
|
32
34
|
}
|
33
|
-
registered_exit_handlers: list[Callable[[], None]] = []
|
34
35
|
|
35
36
|
# SIGQUIT is not available on Windows
|
36
37
|
if hasattr(signal, "SIGQUIT"):
|
37
38
|
SIGNAL_TO_EXIT_CODE[signal.SIGQUIT] = ExitCode.GRACEFUL_EXIT_SIGQUIT
|
38
39
|
|
39
40
|
|
40
|
-
def
|
41
|
+
def register_signal_handlers(
|
41
42
|
event_type: EventType,
|
42
43
|
exit_message: Optional[str] = None,
|
43
44
|
grpc_servers: Optional[list[Server]] = None,
|
@@ -63,7 +64,21 @@ def register_exit_handlers(
|
|
63
64
|
Additional exit handlers can be added using `add_exit_handler`.
|
64
65
|
"""
|
65
66
|
default_handlers: dict[int, Callable[[int, FrameType], None]] = {}
|
66
|
-
|
67
|
+
|
68
|
+
def _wait_to_stop() -> None:
|
69
|
+
if grpc_servers is not None:
|
70
|
+
for grpc_server in grpc_servers:
|
71
|
+
grpc_server.stop(grace=1)
|
72
|
+
|
73
|
+
if bckg_threads is not None:
|
74
|
+
for bckg_thread in bckg_threads:
|
75
|
+
bckg_thread.join()
|
76
|
+
|
77
|
+
# Ensure that `_wait_to_stop` is the last handler called on exit
|
78
|
+
add_exit_handler(_wait_to_stop)
|
79
|
+
|
80
|
+
for handler in exit_handlers or []:
|
81
|
+
add_exit_handler(handler)
|
67
82
|
|
68
83
|
def graceful_exit_handler(signalnum: int, _frame: FrameType) -> None:
|
69
84
|
"""Exit handler to be registered with `signal.signal`.
|
@@ -74,17 +89,6 @@ def register_exit_handlers(
|
|
74
89
|
# Reset to default handler
|
75
90
|
signal.signal(signalnum, default_handlers[signalnum]) # type: ignore
|
76
91
|
|
77
|
-
for handler in registered_exit_handlers:
|
78
|
-
handler()
|
79
|
-
|
80
|
-
if grpc_servers is not None:
|
81
|
-
for grpc_server in grpc_servers:
|
82
|
-
grpc_server.stop(grace=1)
|
83
|
-
|
84
|
-
if bckg_threads is not None:
|
85
|
-
for bckg_thread in bckg_threads:
|
86
|
-
bckg_thread.join()
|
87
|
-
|
88
92
|
# Setup things for graceful exit
|
89
93
|
flwr_exit(
|
90
94
|
code=SIGNAL_TO_EXIT_CODE[signalnum],
|
@@ -96,24 +100,3 @@ def register_exit_handlers(
|
|
96
100
|
for sig in SIGNAL_TO_EXIT_CODE:
|
97
101
|
default_handler = signal.signal(sig, graceful_exit_handler) # type: ignore
|
98
102
|
default_handlers[sig] = default_handler # type: ignore
|
99
|
-
|
100
|
-
|
101
|
-
def add_exit_handler(exit_handler: Callable[[], None]) -> None:
|
102
|
-
"""Add an exit handler to be called on graceful exit.
|
103
|
-
|
104
|
-
This function allows you to register additional exit handlers
|
105
|
-
that will be executed when the application exits gracefully,
|
106
|
-
if `register_exit_handlers` was called.
|
107
|
-
|
108
|
-
Parameters
|
109
|
-
----------
|
110
|
-
exit_handler : Callable[[], None]
|
111
|
-
A callable that takes no arguments and performs cleanup or
|
112
|
-
other actions before the application exits.
|
113
|
-
|
114
|
-
Notes
|
115
|
-
-----
|
116
|
-
This method is not thread-safe, and it allows you to add the
|
117
|
-
same exit handler multiple times.
|
118
|
-
"""
|
119
|
-
registered_exit_handlers.append(exit_handler)
|
flwr/common/grpc.py
CHANGED
@@ -23,9 +23,6 @@ from logging import DEBUG, ERROR
|
|
23
23
|
from typing import Any, Callable, Optional
|
24
24
|
|
25
25
|
import grpc
|
26
|
-
from grpc_health.v1.health_pb2_grpc import add_HealthServicer_to_server
|
27
|
-
|
28
|
-
from flwr.supercore.grpc_health import SimpleHealthServicer
|
29
26
|
|
30
27
|
from .address import is_port_in_use
|
31
28
|
from .logger import log
|
@@ -109,7 +106,6 @@ def generic_create_grpc_server( # pylint: disable=too-many-arguments, R0914, R0
|
|
109
106
|
keepalive_time_ms: int = 210000,
|
110
107
|
certificates: Optional[tuple[bytes, bytes, bytes]] = None,
|
111
108
|
interceptors: Optional[Sequence[grpc.ServerInterceptor]] = None,
|
112
|
-
health_servicer: Optional[Any] = None,
|
113
109
|
) -> grpc.Server:
|
114
110
|
"""Create a gRPC server with a single servicer.
|
115
111
|
|
@@ -157,10 +153,6 @@ def generic_create_grpc_server( # pylint: disable=too-many-arguments, R0914, R0
|
|
157
153
|
* server private key.
|
158
154
|
interceptors : Optional[Sequence[grpc.ServerInterceptor]] (default: None)
|
159
155
|
A list of gRPC interceptors.
|
160
|
-
health_servicer : Optional[Any] (default: None)
|
161
|
-
An optional health servicer to add to the server. If provided, it should be an
|
162
|
-
instance of a class that inherits the `HealthServicer` class.
|
163
|
-
If None is provided, `SimpleHealthServicer` will be used by default.
|
164
156
|
|
165
157
|
Returns
|
166
158
|
-------
|
@@ -211,9 +203,6 @@ def generic_create_grpc_server( # pylint: disable=too-many-arguments, R0914, R0
|
|
211
203
|
)
|
212
204
|
add_servicer_to_server_fn(servicer, server)
|
213
205
|
|
214
|
-
# Enable health service
|
215
|
-
add_HealthServicer_to_server(health_servicer or SimpleHealthServicer(), server)
|
216
|
-
|
217
206
|
if certificates is not None:
|
218
207
|
if not valid_certificates(certificates):
|
219
208
|
sys.exit(1)
|
flwr/common/inflatable_utils.py
CHANGED
flwr/compat/server/app.py
CHANGED
@@ -22,7 +22,7 @@ from typing import Optional
|
|
22
22
|
from flwr.common import GRPC_MAX_MESSAGE_LENGTH, EventType, event
|
23
23
|
from flwr.common.address import parse_address
|
24
24
|
from flwr.common.constant import FLEET_API_GRPC_BIDI_DEFAULT_ADDRESS
|
25
|
-
from flwr.common.
|
25
|
+
from flwr.common.exit import register_signal_handlers
|
26
26
|
from flwr.common.logger import log, warn_deprecated_feature
|
27
27
|
from flwr.server.client_manager import ClientManager
|
28
28
|
from flwr.server.history import History
|
@@ -154,7 +154,7 @@ def start_server( # pylint: disable=too-many-arguments,too-many-locals
|
|
154
154
|
)
|
155
155
|
|
156
156
|
# Graceful shutdown
|
157
|
-
|
157
|
+
register_signal_handlers(
|
158
158
|
event_type=EventType.START_SERVER_LEAVE,
|
159
159
|
exit_message="Flower server terminated gracefully.",
|
160
160
|
grpc_servers=[grpc_server],
|
flwr/server/app.py
CHANGED
@@ -57,8 +57,7 @@ from flwr.common.constant import (
|
|
57
57
|
ExecPluginType,
|
58
58
|
)
|
59
59
|
from flwr.common.event_log_plugin import EventLogWriterPlugin
|
60
|
-
from flwr.common.exit import ExitCode, flwr_exit
|
61
|
-
from flwr.common.exit_handlers import register_exit_handlers
|
60
|
+
from flwr.common.exit import ExitCode, flwr_exit, register_signal_handlers
|
62
61
|
from flwr.common.grpc import generic_create_grpc_server
|
63
62
|
from flwr.common.logger import log
|
64
63
|
from flwr.common.secure_aggregation.crypto.symmetric_encryption import (
|
@@ -70,6 +69,7 @@ from flwr.proto.fleet_pb2_grpc import ( # pylint: disable=E0611
|
|
70
69
|
from flwr.proto.grpcadapter_pb2_grpc import add_GrpcAdapterServicer_to_server
|
71
70
|
from flwr.server.fleet_event_log_interceptor import FleetEventLogInterceptor
|
72
71
|
from flwr.supercore.ffs import FfsFactory
|
72
|
+
from flwr.supercore.grpc_health import add_args_health, run_health_server_grpc_no_tls
|
73
73
|
from flwr.supercore.object_store import ObjectStoreFactory
|
74
74
|
from flwr.superlink.servicer.control import run_control_api_grpc
|
75
75
|
|
@@ -176,6 +176,9 @@ def run_superlink() -> None:
|
|
176
176
|
serverappio_address, _, _ = _format_address(args.serverappio_api_address)
|
177
177
|
control_address, _, _ = _format_address(args.control_api_address)
|
178
178
|
simulationio_address, _, _ = _format_address(args.simulationio_api_address)
|
179
|
+
health_server_address = None
|
180
|
+
if args.health_server_address is not None:
|
181
|
+
health_server_address, _, _ = _format_address(args.health_server_address)
|
179
182
|
|
180
183
|
# Obtain certificates
|
181
184
|
certificates = try_obtain_server_certificates(args)
|
@@ -352,8 +355,13 @@ def run_superlink() -> None:
|
|
352
355
|
# pylint: disable-next=consider-using-with
|
353
356
|
subprocess.Popen(command)
|
354
357
|
|
358
|
+
# Launch gRPC health server
|
359
|
+
if health_server_address is not None:
|
360
|
+
health_server = run_health_server_grpc_no_tls(health_server_address)
|
361
|
+
grpc_servers.append(health_server)
|
362
|
+
|
355
363
|
# Graceful shutdown
|
356
|
-
|
364
|
+
register_signal_handlers(
|
357
365
|
event_type=EventType.RUN_SUPERLINK_LEAVE,
|
358
366
|
exit_message="SuperLink terminated gracefully.",
|
359
367
|
grpc_servers=grpc_servers,
|
@@ -610,6 +618,7 @@ def _parse_args_run_superlink() -> argparse.ArgumentParser:
|
|
610
618
|
_add_args_fleet_api(parser=parser)
|
611
619
|
_add_args_control_api(parser=parser)
|
612
620
|
_add_args_simulationio_api(parser=parser)
|
621
|
+
add_args_health(parser=parser)
|
613
622
|
|
614
623
|
return parser
|
615
624
|
|