flwr 1.18.0__py3-none-any.whl → 1.20.0__py3-none-any.whl

flwr/common/constant.py

@@ -55,13 +55,14 @@ EXEC_API_DEFAULT_SERVER_ADDRESS = f"{SERVER_OCTET}:{EXEC_API_PORT}"
 SIMULATIONIO_API_DEFAULT_SERVER_ADDRESS = f"{SERVER_OCTET}:{SIMULATIONIO_PORT}"
 SIMULATIONIO_API_DEFAULT_CLIENT_ADDRESS = f"{CLIENT_OCTET}:{SIMULATIONIO_PORT}"
 
-# Constants for ping
-PING_DEFAULT_INTERVAL = 30
-PING_CALL_TIMEOUT = 5
-PING_BASE_MULTIPLIER = 0.8
-PING_RANDOM_RANGE = (-0.1, 0.1)
-PING_MAX_INTERVAL = 1e300
-PING_PATIENCE = 2
+# Constants for heartbeat
+HEARTBEAT_DEFAULT_INTERVAL = 30
+HEARTBEAT_CALL_TIMEOUT = 5
+HEARTBEAT_BASE_MULTIPLIER = 0.8
+HEARTBEAT_RANDOM_RANGE = (-0.1, 0.1)
+HEARTBEAT_MAX_INTERVAL = 1e300
+HEARTBEAT_PATIENCE = 2
+RUN_FAILURE_DETAILS_NO_HEARTBEAT = "No heartbeat received from the run."
 
 # IDs
 RUN_ID_NUM_BYTES = 8
@@ -73,6 +74,7 @@ FAB_ALLOWED_EXTENSIONS = {".py", ".toml", ".md"}
 FAB_CONFIG_FILE = "pyproject.toml"
 FAB_DATE = (2024, 10, 1, 0, 0, 0)
 FAB_HASH_TRUNCATION = 8
+FAB_MAX_SIZE = 10 * 1024 * 1024  # 10 MB
 FLWR_DIR = ".flwr"  # The default Flower directory: ~/.flwr/
 FLWR_HOME = "FLWR_HOME"  # If set, override the default Flower directory
 
@@ -114,16 +116,45 @@ AUTH_TYPE_YAML_KEY = "auth_type"  # For key name in YAML file
 ACCESS_TOKEN_KEY = "flwr-oidc-access-token"
 REFRESH_TOKEN_KEY = "flwr-oidc-refresh-token"
 
+# Constants for user authorization
+AUTHZ_TYPE_YAML_KEY = "authz_type"  # For key name in YAML file
+
 # Constants for node authentication
 PUBLIC_KEY_HEADER = "flwr-public-key-bin"  # Must end with "-bin" for binary data
 SIGNATURE_HEADER = "flwr-signature-bin"  # Must end with "-bin" for binary data
 TIMESTAMP_HEADER = "flwr-timestamp"
-TIMESTAMP_TOLERANCE = 10  # General tolerance for timestamp verification
+TIMESTAMP_TOLERANCE = 300  # General tolerance for timestamp verification
 SYSTEM_TIME_TOLERANCE = 5  # Allowance for system time drift
 
+# Constants for grpc retry
+GRPC_RETRY_MAX_DELAY = 20  # Maximum delay duration between two consecutive retries.
+
 # Constants for ArrayRecord
 GC_THRESHOLD = 200_000_000  # 200 MB
 
+# Constants for Inflatable
+HEAD_BODY_DIVIDER = b"\x00"
+HEAD_VALUE_DIVIDER = " "
+MAX_ARRAY_CHUNK_SIZE = 20_971_520  # 20 MB
+
+# Constants for serialization
+INT64_MAX_VALUE = 9223372036854775807  # (1 << 63) - 1
+
+# Constants for `flwr-serverapp` and `flwr-clientapp` CLI commands
+FLWR_APP_TOKEN_LENGTH = 128  # Length of the token used
+
+# Constants for object pushing and pulling
+MAX_CONCURRENT_PUSHES = 8  # Default maximum number of concurrent pushes
+MAX_CONCURRENT_PULLS = 8  # Default maximum number of concurrent pulls
+PULL_MAX_TIME = 7200  # Default maximum time to wait for pulling objects
+PULL_MAX_TRIES_PER_OBJECT = 500  # Default maximum number of tries to pull an object
+PULL_INITIAL_BACKOFF = 1  # Initial backoff time for pulling objects
+PULL_BACKOFF_CAP = 10  # Maximum backoff time for pulling objects
+
+
+# ExecServicer constants
+RUN_ID_NOT_FOUND_MESSAGE = "Run ID not found"
+
 
 class MessageType:
     """Message type."""

flwr/common/event_log_plugin/event_log_plugin.py

@@ -21,7 +21,7 @@ from typing import Optional, Union
 import grpc
 from google.protobuf.message import Message as GrpcMessage
 
-from flwr.common.typing import LogEntry, UserInfo
+from flwr.common.typing import AccountInfo, LogEntry
 
 
 class EventLogWriterPlugin(ABC):
@@ -36,7 +36,7 @@ class EventLogWriterPlugin(ABC):
         self,
         request: GrpcMessage,
         context: grpc.ServicerContext,
-        user_info: Optional[UserInfo],
+        account_info: Optional[AccountInfo],
         method_name: str,
     ) -> LogEntry:
         """Compose pre-event log entry from the provided request and context."""
@@ -46,7 +46,7 @@ class EventLogWriterPlugin(ABC):
         self,
         request: GrpcMessage,
         context: grpc.ServicerContext,
-        user_info: Optional[UserInfo],
+        account_info: Optional[AccountInfo],
         method_name: str,
         response: Optional[Union[GrpcMessage, BaseException]],
     ) -> LogEntry:

flwr/common/exit/exit_code.py

@@ -29,6 +29,9 @@ class ExitCode:
 
     # SuperLink-specific exit codes (100-199)
     SUPERLINK_THREAD_CRASH = 100
+    SUPERLINK_LICENSE_INVALID = 101
+    SUPERLINK_LICENSE_MISSING = 102
+    SUPERLINK_LICENSE_URL_INVALID = 103
 
     # ServerApp-specific exit codes (200-299)
 
@@ -60,6 +63,18 @@ EXIT_CODE_HELP = {
     ExitCode.GRACEFUL_EXIT_SIGTERM: "",
     # SuperLink-specific exit codes (100-199)
     ExitCode.SUPERLINK_THREAD_CRASH: "An important background thread has crashed.",
+    ExitCode.SUPERLINK_LICENSE_INVALID: (
+        "The license is invalid or has expired. "
+        "Please contact `hello@flower.ai` for assistance."
+    ),
+    ExitCode.SUPERLINK_LICENSE_MISSING: (
+        "The license is missing. Please specify the license key by setting the "
+        "environment variable `FLWR_LICENSE_KEY`."
+    ),
+    ExitCode.SUPERLINK_LICENSE_URL_INVALID: (
+        "The license URL is invalid. Please ensure that the `FLWR_LICENSE_URL` "
+        "environment variable is set to a valid URL."
+    ),
     # ServerApp-specific exit codes (200-299)
     # SuperNode-specific exit codes (300-399)
     ExitCode.SUPERNODE_REST_ADDRESS_INVALID: (
@@ -72,7 +87,7 @@ EXIT_CODE_HELP = {
         "to be provided (providing only one of them is not sufficient)."
     ),
     ExitCode.SUPERNODE_NODE_AUTH_KEYS_INVALID: (
-        "Node uthentication requires elliptic curve private and public key pair. "
+        "Node authentication requires elliptic curve private and public key pair. "
         "Please ensure that the file path points to a valid private/public key "
         "file and try again."
     ),

flwr/common/exit_handlers.py

@@ -30,6 +30,7 @@ SIGNAL_TO_EXIT_CODE: dict[int, int] = {
     signal.SIGINT: ExitCode.GRACEFUL_EXIT_SIGINT,
     signal.SIGTERM: ExitCode.GRACEFUL_EXIT_SIGTERM,
 }
+registered_exit_handlers: list[Callable[[], None]] = []
 
 # SIGQUIT is not available on Windows
 if hasattr(signal, "SIGQUIT"):
@@ -41,6 +42,7 @@ def register_exit_handlers(
     exit_message: Optional[str] = None,
     grpc_servers: Optional[list[Server]] = None,
     bckg_threads: Optional[list[Thread]] = None,
+    exit_handlers: Optional[list[Callable[[], None]]] = None,
 ) -> None:
     """Register exit handlers for `SIGINT`, `SIGTERM` and `SIGQUIT` signals.
 
@@ -56,8 +58,12 @@ def register_exit_handlers(
     bckg_threads: Optional[List[Thread]] (default: None)
         An optional list of threads that need to be gracefully
         terminated before exiting.
+    exit_handlers: Optional[List[Callable[[], None]]] (default: None)
+        An optional list of exit handlers to be called before exiting.
+        Additional exit handlers can be added using `add_exit_handler`.
     """
     default_handlers: dict[int, Callable[[int, FrameType], None]] = {}
+    registered_exit_handlers.extend(exit_handlers or [])
 
     def graceful_exit_handler(signalnum: int, _frame: FrameType) -> None:
         """Exit handler to be registered with `signal.signal`.
@@ -68,6 +74,9 @@ def register_exit_handlers(
         # Reset to default handler
         signal.signal(signalnum, default_handlers[signalnum])  # type: ignore
 
+        for handler in registered_exit_handlers:
+            handler()
+
         if grpc_servers is not None:
             for grpc_server in grpc_servers:
                 grpc_server.stop(grace=1)
@@ -87,3 +96,24 @@ def register_exit_handlers(
     for sig in SIGNAL_TO_EXIT_CODE:
         default_handler = signal.signal(sig, graceful_exit_handler)  # type: ignore
         default_handlers[sig] = default_handler  # type: ignore
+
+
+def add_exit_handler(exit_handler: Callable[[], None]) -> None:
+    """Add an exit handler to be called on graceful exit.
+
+    This function allows you to register additional exit handlers
+    that will be executed when the application exits gracefully,
+    if `register_exit_handlers` was called.
+
+    Parameters
+    ----------
+    exit_handler : Callable[[], None]
+        A callable that takes no arguments and performs cleanup or
+        other actions before the application exits.
+
+    Notes
+    -----
+    This method is not thread-safe, and it allows you to add the
+    same exit handler multiple times.
+    """
+    registered_exit_handlers.append(exit_handler)

flwr/common/grpc.py

@@ -23,6 +23,9 @@ from logging import DEBUG, ERROR
 from typing import Any, Callable, Optional
 
 import grpc
+from grpc_health.v1.health_pb2_grpc import add_HealthServicer_to_server
+
+from flwr.supercore.grpc_health import SimpleHealthServicer
 
 from .address import is_port_in_use
 from .logger import log
@@ -98,7 +101,7 @@ def valid_certificates(certificates: tuple[bytes, bytes, bytes]) -> bool:
     return is_valid
 
 
-def generic_create_grpc_server(  # pylint: disable=too-many-arguments,R0917
+def generic_create_grpc_server(  # pylint: disable=too-many-arguments, R0914, R0917
     servicer_and_add_fn: tuple[Any, AddServicerToServerFn],
     server_address: str,
     max_concurrent_workers: int = 1000,
@@ -106,6 +109,7 @@ def generic_create_grpc_server(  # pylint: disable=too-many-arguments,R0917
     keepalive_time_ms: int = 210000,
     certificates: Optional[tuple[bytes, bytes, bytes]] = None,
     interceptors: Optional[Sequence[grpc.ServerInterceptor]] = None,
+    health_servicer: Optional[Any] = None,
 ) -> grpc.Server:
     """Create a gRPC server with a single servicer.
 
@@ -153,6 +157,10 @@ def generic_create_grpc_server(  # pylint: disable=too-many-arguments,R0917
             * server private key.
     interceptors : Optional[Sequence[grpc.ServerInterceptor]] (default: None)
         A list of gRPC interceptors.
+    health_servicer : Optional[Any] (default: None)
+        An optional health servicer to add to the server. If provided, it should be an
+        instance of a class that inherits the `HealthServicer` class.
+        If None is provided, `SimpleHealthServicer` will be used by default.
 
     Returns
     -------
@@ -203,6 +211,9 @@ def generic_create_grpc_server(  # pylint: disable=too-many-arguments,R0917
     )
     add_servicer_to_server_fn(servicer, server)
 
+    # Enable health service
+    add_HealthServicer_to_server(health_servicer or SimpleHealthServicer(), server)
+
     if certificates is not None:
         if not valid_certificates(certificates):
             sys.exit(1)

flwr/common/heartbeat.py

@@ -0,0 +1,165 @@
+# Copyright 2025 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Heartbeat sender."""
+
+
+import random
+import threading
+from typing import Callable, Union
+
+import grpc
+
+# pylint: disable=E0611
+from flwr.proto.heartbeat_pb2 import SendAppHeartbeatRequest
+from flwr.proto.serverappio_pb2_grpc import ServerAppIoStub
+from flwr.proto.simulationio_pb2_grpc import SimulationIoStub
+
+# pylint: enable=E0611
+from .constant import (
+    HEARTBEAT_BASE_MULTIPLIER,
+    HEARTBEAT_CALL_TIMEOUT,
+    HEARTBEAT_DEFAULT_INTERVAL,
+    HEARTBEAT_RANDOM_RANGE,
+)
+from .retry_invoker import RetryInvoker, exponential
+
+
+class HeartbeatFailure(Exception):
+    """Exception raised when a heartbeat fails."""
+
+
+class HeartbeatSender:
+    """Periodically send heartbeat signals to a server in a background thread.
+
+    This class uses the provided `heartbeat_fn` to send heartbeats. If a heartbeat
+    attempt fails, it will be retried using an exponential backoff strategy.
+
+    Parameters
+    ----------
+    heartbeat_fn : Callable[[], bool]
+        Function used to send a heartbeat signal. It should return True if the heartbeat
+        succeeds, or False if it fails. Any internal exceptions (e.g., gRPC errors)
+        should be handled within this function to ensure boolean return values.
+    """
+
+    def __init__(
+        self,
+        heartbeat_fn: Callable[[], bool],
+    ) -> None:
+        self.heartbeat_fn = heartbeat_fn
+        self._stop_event = threading.Event()
+        self._thread = threading.Thread(target=self._run, daemon=True)
+        self._retry_invoker = RetryInvoker(
+            lambda: exponential(max_delay=20),
+            HeartbeatFailure,  # The only exception we want to retry on
+            max_tries=None,
+            max_time=None,
+            # Allow the stop event to interrupt the wait
+            wait_function=self._stop_event.wait,  # type: ignore
+        )
+
+    def start(self) -> None:
+        """Start the heartbeat sender."""
+        if self._thread.is_alive():
+            raise RuntimeError("Heartbeat sender is already running.")
+        if self._stop_event.is_set():
+            raise RuntimeError("Cannot start a stopped heartbeat sender.")
+        self._thread.start()
+
+    def stop(self) -> None:
+        """Stop the heartbeat sender."""
+        if not self._thread.is_alive():
+            raise RuntimeError("Heartbeat sender is not running.")
+        self._stop_event.set()
+        self._thread.join()
+
+    @property
+    def is_running(self) -> bool:
+        """Return True if the heartbeat sender is running, False otherwise."""
+        return self._thread.is_alive() and not self._stop_event.is_set()
+
+    def _run(self) -> None:
+        """Periodically send heartbeats until stopped."""
+        while not self._stop_event.is_set():
+            # Attempt to send a heartbeat with retry on failure
+            self._retry_invoker.invoke(self._heartbeat)
+
+            # Calculate the interval for the next heartbeat
+            # Formula: next_interval = (interval - timeout) * random.uniform(0.7, 0.9)
+            rd = random.uniform(*HEARTBEAT_RANDOM_RANGE)
+            next_interval: float = HEARTBEAT_DEFAULT_INTERVAL - HEARTBEAT_CALL_TIMEOUT
+            next_interval *= HEARTBEAT_BASE_MULTIPLIER + rd
+
+            # Wait for the calculated interval or exit early if stopped
+            self._stop_event.wait(next_interval)
+
+    def _heartbeat(self) -> None:
+        """Send a single heartbeat and raise an exception if it fails.
+
+        Call the provided `heartbeat_fn`. If the function returns False,
+        a `HeartbeatFailure` exception is raised to trigger the retry mechanism.
+        """
+        if not self._stop_event.is_set():
+            if not self.heartbeat_fn():
+                raise HeartbeatFailure
+
+
+def get_grpc_app_heartbeat_fn(
+    stub: Union[ServerAppIoStub, SimulationIoStub],
+    run_id: int,
+    *,
+    failure_message: str,
+) -> Callable[[], bool]:
+    """Get the function to send a heartbeat to gRPC endpoint.
+
+    This function is for app heartbeats only. It is not used for node heartbeats.
+
+    Parameters
+    ----------
+    stub : Union[ServerAppIoStub, SimulationIoStub]
+        gRPC stub to send the heartbeat.
+    run_id : int
+        The run ID to use in the heartbeat request.
+    failure_message : str
+        Error message to raise if the heartbeat fails.
+
+    Returns
+    -------
+    Callable[[], bool]
+        Function that sends a heartbeat to the gRPC endpoint.
+    """
+    # Construct the heartbeat request
+    req = SendAppHeartbeatRequest(
+        run_id=run_id, heartbeat_interval=HEARTBEAT_DEFAULT_INTERVAL
+    )
+
+    def fn() -> bool:
+        # Call ServerAppIo API
+        try:
+            res = stub.SendAppHeartbeat(req)
+        except grpc.RpcError as e:
+            status_code = e.code()
+            if status_code == grpc.StatusCode.UNAVAILABLE:
+                return False
+            if status_code == grpc.StatusCode.DEADLINE_EXCEEDED:
+                return False
+            raise
+
+        # Check if not successful
+        if not res.success:
+            raise RuntimeError(failure_message)
+        return True
+
+    return fn