flwr-nightly 1.8.0.dev20240328__py3-none-any.whl → 1.8.0.dev20240401__py3-none-any.whl

flwr/client/app.py

@@ -36,7 +36,7 @@ from flwr.common.constant import (
     TRANSPORT_TYPES,
 )
 from flwr.common.exit_handlers import register_exit_handlers
-from flwr.common.logger import log, warn_deprecated_feature, warn_experimental_feature
+from flwr.common.logger import log, warn_deprecated_feature
 from flwr.common.message import Error
 from flwr.common.object_ref import load_app, validate
 from flwr.common.retry_invoker import RetryInvoker, exponential
@@ -385,8 +385,6 @@ def _start_client_internal(
             return ClientApp(client_fn=client_fn)
 
         load_client_app_fn = _load_client_app
-    else:
-        warn_experimental_feature("`load_client_app_fn`")
 
     # At this point, only `load_client_app_fn` should be used
     # Both `client` and `client_fn` must not be used directly
@@ -397,7 +395,7 @@ def _start_client_internal(
     )
 
     retry_invoker = RetryInvoker(
-        wait_factory=exponential,
+        wait_gen_factory=exponential,
         recoverable_exceptions=connection_error_type,
         max_tries=max_retries,
         max_time=max_wait_time,

flwr/client/client_app.py

@@ -23,6 +23,7 @@ from flwr.client.message_handler.message_handler import (
 from flwr.client.mod.utils import make_ffn
 from flwr.client.typing import ClientFn, Mod
 from flwr.common import Context, Message, MessageType
+from flwr.common.logger import warn_preview_feature
 
 from .typing import ClientAppCallable
 
@@ -123,6 +124,8 @@ class ClientApp:
             if self._call:
                 raise _registration_error(MessageType.TRAIN)
 
+            warn_preview_feature("ClientApp-register-train-function")
+
             # Register provided function with the ClientApp object
             # Wrap mods around the wrapped step function
             self._train = make_ffn(train_fn, self._mods)
@@ -151,6 +154,8 @@ class ClientApp:
             if self._call:
                 raise _registration_error(MessageType.EVALUATE)
 
+            warn_preview_feature("ClientApp-register-evaluate-function")
+
             # Register provided function with the ClientApp object
             # Wrap mods around the wrapped step function
             self._evaluate = make_ffn(evaluate_fn, self._mods)
@@ -179,6 +184,8 @@ class ClientApp:
             if self._call:
                 raise _registration_error(MessageType.QUERY)
 
+            warn_preview_feature("ClientApp-register-query-function")
+
             # Register provided function with the ClientApp object
             # Wrap mods around the wrapped step function
             self._query = make_ffn(query_fn, self._mods)

flwr/client/grpc_rere_client/connection.py

@@ -15,23 +15,34 @@
 """Contextmanager for a gRPC request-response channel to the Flower server."""
 
 
+import random
+import threading
 from contextlib import contextmanager
 from copy import copy
 from logging import DEBUG, ERROR
 from pathlib import Path
-from typing import Callable, Dict, Iterator, Optional, Tuple, Union, cast
+from typing import Callable, Iterator, Optional, Tuple, Union, cast
 
+from flwr.client.heartbeat import start_ping_loop
 from flwr.client.message_handler.message_handler import validate_out_message
 from flwr.client.message_handler.task_handler import get_task_ins, validate_task_ins
 from flwr.common import GRPC_MAX_MESSAGE_LENGTH
+from flwr.common.constant import (
+    PING_BASE_MULTIPLIER,
+    PING_CALL_TIMEOUT,
+    PING_DEFAULT_INTERVAL,
+    PING_RANDOM_RANGE,
+)
 from flwr.common.grpc import create_channel
-from flwr.common.logger import log, warn_experimental_feature
+from flwr.common.logger import log
 from flwr.common.message import Message, Metadata
 from flwr.common.retry_invoker import RetryInvoker
 from flwr.common.serde import message_from_taskins, message_to_taskres
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     DeleteNodeRequest,
+    PingRequest,
+    PingResponse,
     PullTaskInsRequest,
     PushTaskResRequest,
 )
@@ -39,9 +50,6 @@ from flwr.proto.fleet_pb2_grpc import FleetStub  # pylint: disable=E0611
 from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
 from flwr.proto.task_pb2 import TaskIns  # pylint: disable=E0611
 
-KEY_NODE = "node"
-KEY_METADATA = "in_message_metadata"
-
 
 def on_channel_state_change(channel_connectivity: str) -> None:
     """Log channel connectivity."""
@@ -49,7 +57,7 @@ def on_channel_state_change(channel_connectivity: str) -> None:
 
 
 @contextmanager
-def grpc_request_response(
+def grpc_request_response(  # pylint: disable=R0914, R0915
     server_address: str,
     insecure: bool,
     retry_invoker: RetryInvoker,
@@ -95,8 +103,6 @@ def grpc_request_response(
     create_node : Optional[Callable]
     delete_node : Optional[Callable]
     """
-    warn_experimental_feature("`grpc-rere`")
-
     if isinstance(root_certificates, str):
         root_certificates = Path(root_certificates).read_bytes()
 
@@ -107,47 +113,81 @@ def grpc_request_response(
         max_message_length=max_message_length,
     )
     channel.subscribe(on_channel_state_change)
-    stub = FleetStub(channel)
 
-    # Necessary state to validate messages to be sent
-    state: Dict[str, Optional[Metadata]] = {KEY_METADATA: None}
-
-    # Enable create_node and delete_node to store node
-    node_store: Dict[str, Optional[Node]] = {KEY_NODE: None}
+    # Shared variables for inner functions
+    stub = FleetStub(channel)
+    metadata: Optional[Metadata] = None
+    node: Optional[Node] = None
+    ping_thread: Optional[threading.Thread] = None
+    ping_stop_event = threading.Event()
 
     ###########################################################################
-    # receive/send functions
+    # ping/create_node/delete_node/receive/send functions
     ###########################################################################
 
+    def ping() -> None:
+        # Get Node
+        if node is None:
+            log(ERROR, "Node instance missing")
+            return
+
+        # Construct the ping request
+        req = PingRequest(node=node, ping_interval=PING_DEFAULT_INTERVAL)
+
+        # Call FleetAPI
+        res: PingResponse = stub.Ping(req, timeout=PING_CALL_TIMEOUT)
+
+        # Check if success
+        if not res.success:
+            raise RuntimeError("Ping failed unexpectedly.")
+
+        # Wait
+        rd = random.uniform(*PING_RANDOM_RANGE)
+        next_interval: float = PING_DEFAULT_INTERVAL - PING_CALL_TIMEOUT
+        next_interval *= PING_BASE_MULTIPLIER + rd
+        if not ping_stop_event.is_set():
+            ping_stop_event.wait(next_interval)
+
     def create_node() -> None:
         """Set create_node."""
+        # Call FleetAPI
         create_node_request = CreateNodeRequest()
         create_node_response = retry_invoker.invoke(
             stub.CreateNode,
             request=create_node_request,
         )
-        node_store[KEY_NODE] = create_node_response.node
+
+        # Remember the node and the ping-loop thread
+        nonlocal node, ping_thread
+        node = cast(Node, create_node_response.node)
+        ping_thread = start_ping_loop(ping, ping_stop_event)
 
     def delete_node() -> None:
         """Set delete_node."""
         # Get Node
-        if node_store[KEY_NODE] is None:
+        nonlocal node
+        if node is None:
             log(ERROR, "Node instance missing")
             return
-        node: Node = cast(Node, node_store[KEY_NODE])
 
+        # Stop the ping-loop thread
+        ping_stop_event.set()
+        if ping_thread is not None:
+            ping_thread.join()
+
+        # Call FleetAPI
         delete_node_request = DeleteNodeRequest(node=node)
         retry_invoker.invoke(stub.DeleteNode, request=delete_node_request)
 
-        del node_store[KEY_NODE]
+        # Cleanup
+        node = None
 
     def receive() -> Optional[Message]:
         """Receive next task from server."""
         # Get Node
-        if node_store[KEY_NODE] is None:
+        if node is None:
             log(ERROR, "Node instance missing")
             return None
-        node: Node = cast(Node, node_store[KEY_NODE])
 
         # Request instructions (task) from server
         request = PullTaskInsRequest(node=node)
@@ -167,7 +207,8 @@ def grpc_request_response(
         in_message = message_from_taskins(task_ins) if task_ins else None
 
         # Remember `metadata` of the in message
-        state[KEY_METADATA] = copy(in_message.metadata) if in_message else None
+        nonlocal metadata
+        metadata = copy(in_message.metadata) if in_message else None
 
         # Return the message if available
         return in_message
@@ -175,18 +216,18 @@ def grpc_request_response(
     def send(message: Message) -> None:
         """Send task result back to server."""
         # Get Node
-        if node_store[KEY_NODE] is None:
+        if node is None:
             log(ERROR, "Node instance missing")
             return
 
-        # Get incoming message
-        in_metadata = state[KEY_METADATA]
-        if in_metadata is None:
+        # Get the metadata of the incoming message
+        nonlocal metadata
+        if metadata is None:
             log(ERROR, "No current message")
             return
 
         # Validate out message
-        if not validate_out_message(message, in_metadata):
+        if not validate_out_message(message, metadata):
             log(ERROR, "Invalid out message")
             return
 
@@ -197,7 +238,8 @@ def grpc_request_response(
         request = PushTaskResRequest(task_res_list=[task_res])
         _ = retry_invoker.invoke(stub.PushTaskRes, request)
 
-        state[KEY_METADATA] = None
+        # Cleanup
+        metadata = None
 
     try:
         # Yield methods

flwr/client/heartbeat.py

@@ -0,0 +1,72 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Heartbeat utility functions."""
+
+
+import threading
+from typing import Callable
+
+import grpc
+
+from flwr.common.constant import PING_CALL_TIMEOUT
+from flwr.common.retry_invoker import RetryInvoker, RetryState, exponential
+
+
+def _ping_loop(ping_fn: Callable[[], None], stop_event: threading.Event) -> None:
+    def wait_fn(wait_time: float) -> None:
+        if not stop_event.is_set():
+            stop_event.wait(wait_time)
+
+    def on_backoff(state: RetryState) -> None:
+        err = state.exception
+        if not isinstance(err, grpc.RpcError):
+            return
+        status_code = err.code()
+        # If ping call timeout is triggered
+        if status_code == grpc.StatusCode.DEADLINE_EXCEEDED:
+            # Avoid long wait time.
+            if state.actual_wait is None:
+                return
+            state.actual_wait = max(state.actual_wait - PING_CALL_TIMEOUT, 0.0)
+
+    def wrapped_ping() -> None:
+        if not stop_event.is_set():
+            ping_fn()
+
+    retrier = RetryInvoker(
+        exponential,
+        grpc.RpcError,
+        max_tries=None,
+        max_time=None,
+        on_backoff=on_backoff,
+        wait_function=wait_fn,
+    )
+    while not stop_event.is_set():
+        retrier.invoke(wrapped_ping)
+
+
+def start_ping_loop(
+    ping_fn: Callable[[], None], stop_event: threading.Event
+) -> threading.Thread:
+    """Start a ping loop in a separate thread.
+
+    This function initializes a new thread that runs a ping loop, allowing for
+    asynchronous ping operations. The loop can be terminated through the provided stop
+    event.
+    """
+    thread = threading.Thread(target=_ping_loop, args=(ping_fn, stop_event))
+    thread.start()
+
+    return thread

flwr/client/rest_client/connection.py

@@ -15,16 +15,25 @@
 """Contextmanager for a REST request-response channel to the Flower server."""
 
 
+import random
 import sys
+import threading
 from contextlib import contextmanager
 from copy import copy
 from logging import ERROR, INFO, WARN
-from typing import Callable, Dict, Iterator, Optional, Tuple, Union, cast
+from typing import Callable, Iterator, Optional, Tuple, Union
 
+from flwr.client.heartbeat import start_ping_loop
 from flwr.client.message_handler.message_handler import validate_out_message
 from flwr.client.message_handler.task_handler import get_task_ins, validate_task_ins
 from flwr.common import GRPC_MAX_MESSAGE_LENGTH
-from flwr.common.constant import MISSING_EXTRA_REST
+from flwr.common.constant import (
+    MISSING_EXTRA_REST,
+    PING_BASE_MULTIPLIER,
+    PING_CALL_TIMEOUT,
+    PING_DEFAULT_INTERVAL,
+    PING_RANDOM_RANGE,
+)
 from flwr.common.logger import log
 from flwr.common.message import Message, Metadata
 from flwr.common.retry_invoker import RetryInvoker
@@ -33,6 +42,8 @@ from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     CreateNodeResponse,
     DeleteNodeRequest,
+    PingRequest,
+    PingResponse,
     PullTaskInsRequest,
     PullTaskInsResponse,
     PushTaskResRequest,
@@ -47,19 +58,15 @@ except ModuleNotFoundError:
     sys.exit(MISSING_EXTRA_REST)
 
 
-KEY_NODE = "node"
-KEY_METADATA = "in_message_metadata"
-
-
 PATH_CREATE_NODE: str = "api/v0/fleet/create-node"
 PATH_DELETE_NODE: str = "api/v0/fleet/delete-node"
 PATH_PULL_TASK_INS: str = "api/v0/fleet/pull-task-ins"
 PATH_PUSH_TASK_RES: str = "api/v0/fleet/push-task-res"
+PATH_PING: str = "api/v0/fleet/ping"
 
 
 @contextmanager
-# pylint: disable-next=too-many-statements
-def http_request_response(
+def http_request_response(  # pylint: disable=R0914, R0915
     server_address: str,
     insecure: bool,  # pylint: disable=unused-argument
     retry_invoker: RetryInvoker,
@@ -127,16 +134,71 @@ def http_request_response(
             "must be provided as a string path to the client.",
         )
 
-    # Necessary state to validate messages to be sent
-    state: Dict[str, Optional[Metadata]] = {KEY_METADATA: None}
-
-    # Enable create_node and delete_node to store node
-    node_store: Dict[str, Optional[Node]] = {KEY_NODE: None}
+    # Shared variables for inner functions
+    metadata: Optional[Metadata] = None
+    node: Optional[Node] = None
+    ping_thread: Optional[threading.Thread] = None
+    ping_stop_event = threading.Event()
 
     ###########################################################################
-    # receive/send functions
+    # ping/create_node/delete_node/receive/send functions
     ###########################################################################
 
+    def ping() -> None:
+        # Get Node
+        if node is None:
+            log(ERROR, "Node instance missing")
+            return
+
+        # Construct the ping request
+        req = PingRequest(node=node, ping_interval=PING_DEFAULT_INTERVAL)
+        req_bytes: bytes = req.SerializeToString()
+
+        # Send the request
+        res = requests.post(
+            url=f"{base_url}/{PATH_PING}",
+            headers={
+                "Accept": "application/protobuf",
+                "Content-Type": "application/protobuf",
+            },
+            data=req_bytes,
+            verify=verify,
+            timeout=PING_CALL_TIMEOUT,
+        )
+
+        # Check status code and headers
+        if res.status_code != 200:
+            return
+        if "content-type" not in res.headers:
+            log(
+                WARN,
+                "[Node] POST /%s: missing header `Content-Type`",
+                PATH_PULL_TASK_INS,
+            )
+            return
+        if res.headers["content-type"] != "application/protobuf":
+            log(
+                WARN,
+                "[Node] POST /%s: header `Content-Type` has wrong value",
+                PATH_PULL_TASK_INS,
+            )
+            return
+
+        # Deserialize ProtoBuf from bytes
+        ping_res = PingResponse()
+        ping_res.ParseFromString(res.content)
+
+        # Check if success
+        if not ping_res.success:
+            raise RuntimeError("Ping failed unexpectedly.")
+
+        # Wait
+        rd = random.uniform(*PING_RANDOM_RANGE)
+        next_interval: float = PING_DEFAULT_INTERVAL - PING_CALL_TIMEOUT
+        next_interval *= PING_BASE_MULTIPLIER + rd
+        if not ping_stop_event.is_set():
+            ping_stop_event.wait(next_interval)
+
     def create_node() -> None:
         """Set create_node."""
         create_node_req_proto = CreateNodeRequest()
@@ -175,15 +237,25 @@ def http_request_response(
         # Deserialize ProtoBuf from bytes
         create_node_response_proto = CreateNodeResponse()
         create_node_response_proto.ParseFromString(res.content)
-        # pylint: disable-next=no-member
-        node_store[KEY_NODE] = create_node_response_proto.node
+
+        # Remember the node and the ping-loop thread
+        nonlocal node, ping_thread
+        node = create_node_response_proto.node
+        ping_thread = start_ping_loop(ping, ping_stop_event)
 
     def delete_node() -> None:
         """Set delete_node."""
-        if node_store[KEY_NODE] is None:
+        nonlocal node
+        if node is None:
             log(ERROR, "Node instance missing")
             return
-        node: Node = cast(Node, node_store[KEY_NODE])
+
+        # Stop the ping-loop thread
+        ping_stop_event.set()
+        if ping_thread is not None:
+            ping_thread.join()
+
+        # Send DeleteNode request
         delete_node_req_proto = DeleteNodeRequest(node=node)
         delete_node_req_req_bytes: bytes = delete_node_req_proto.SerializeToString()
         res = retry_invoker.invoke(
@@ -215,13 +287,15 @@ def http_request_response(
                 PATH_PULL_TASK_INS,
             )
 
+        # Cleanup
+        node = None
+
     def receive() -> Optional[Message]:
         """Receive next task from server."""
         # Get Node
-        if node_store[KEY_NODE] is None:
+        if node is None:
             log(ERROR, "Node instance missing")
             return None
-        node: Node = cast(Node, node_store[KEY_NODE])
 
         # Request instructions (task) from server
         pull_task_ins_req_proto = PullTaskInsRequest(node=node)
@@ -273,29 +347,29 @@ def http_request_response(
             task_ins = None
 
         # Return the Message if available
+        nonlocal metadata
         message = None
-        state[KEY_METADATA] = None
         if task_ins is not None:
             message = message_from_taskins(task_ins)
-            state[KEY_METADATA] = copy(message.metadata)
+            metadata = copy(message.metadata)
             log(INFO, "[Node] POST /%s: success", PATH_PULL_TASK_INS)
         return message
 
     def send(message: Message) -> None:
         """Send task result back to server."""
         # Get Node
-        if node_store[KEY_NODE] is None:
+        if node is None:
             log(ERROR, "Node instance missing")
             return
 
         # Get incoming message
-        in_metadata = state[KEY_METADATA]
-        if in_metadata is None:
+        nonlocal metadata
+        if metadata is None:
             log(ERROR, "No current message")
             return
 
         # Validate out message
-        if not validate_out_message(message, in_metadata):
+        if not validate_out_message(message, metadata):
             log(ERROR, "Invalid out message")
             return
 
@@ -321,7 +395,7 @@ def http_request_response(
             timeout=None,
         )
 
-        state[KEY_METADATA] = None
+        metadata = None
 
         # Check status code and headers
         if res.status_code != 200:

flwr/common/constant.py

@@ -36,6 +36,12 @@ TRANSPORT_TYPES = [
     TRANSPORT_TYPE_VCE,
 ]
 
+# Constants for ping
+PING_DEFAULT_INTERVAL = 30
+PING_CALL_TIMEOUT = 5
+PING_BASE_MULTIPLIER = 0.8
+PING_RANDOM_RANGE = (-0.1, 0.1)
+
 
 class MessageType:
     """Message type."""

flwr/common/logger.py

@@ -164,13 +164,13 @@ logger = logging.getLogger(LOGGER_NAME)  # pylint: disable=invalid-name
 log = logger.log  # pylint: disable=invalid-name
 
 
-def warn_experimental_feature(name: str) -> None:
-    """Warn the user when they use an experimental feature."""
+def warn_preview_feature(name: str) -> None:
+    """Warn the user when they use a preview feature."""
     log(
         WARN,
-        """EXPERIMENTAL FEATURE: %s
+        """PREVIEW FEATURE: %s
 
-            This is an experimental feature. It could change significantly or be removed
+            This is a preview feature. It could change significantly or be removed
             entirely in future versions of Flower.
         """,
         name,

flwr/common/retry_invoker.py

@@ -107,7 +107,7 @@ class RetryInvoker:
 
     Parameters
     ----------
-    wait_factory: Callable[[], Generator[float, None, None]]
+    wait_gen_factory: Callable[[], Generator[float, None, None]]
         A generator yielding successive wait times in seconds. If the generator
         is finite, the giveup event will be triggered when the generator raises
         `StopIteration`.
@@ -129,12 +129,12 @@ class RetryInvoker:
         data class object detailing the invocation.
     on_giveup: Optional[Callable[[RetryState], None]] (default: None)
         A callable to be executed in the event that `max_tries` or `max_time` is
-        exceeded, `should_giveup` returns True, or `wait_factory()` generator raises
+        exceeded, `should_giveup` returns True, or `wait_gen_factory()` generator raises
         `StopInteration`. The parameter is a data class object detailing the
         invocation.
     jitter: Optional[Callable[[float], float]] (default: full_jitter)
-        A function of the value yielded by `wait_factory()` returning the actual time
-        to wait. This function helps distribute wait times stochastically to avoid
+        A function of the value yielded by `wait_gen_factory()` returning the actual
+        time to wait. This function helps distribute wait times stochastically to avoid
         timing collisions across concurrent clients. Wait times are jittered by
         default using the `full_jitter` function. To disable jittering, pass
         `jitter=None`.
@@ -142,6 +142,13 @@ class RetryInvoker:
         A function accepting an exception instance, returning whether or not
         to give up prematurely before other give-up conditions are evaluated.
         If set to None, the strategy is to never give up prematurely.
+    wait_function: Optional[Callable[[float], None]] (default: None)
+        A function that defines how to wait between retry attempts. It accepts
+        one argument, the wait time in seconds, allowing the use of various waiting
+        mechanisms (e.g., asynchronous waits or event-based synchronization) suitable
+        for different execution environments. If set to `None`, the `wait_function`
+        defaults to `time.sleep`, which is ideal for synchronous operations. Custom
+        functions should manage execution flow to prevent blocking or interference.
 
     Examples
     --------
@@ -159,7 +166,7 @@ class RetryInvoker:
     # pylint: disable-next=too-many-arguments
     def __init__(
         self,
-        wait_factory: Callable[[], Generator[float, None, None]],
+        wait_gen_factory: Callable[[], Generator[float, None, None]],
         recoverable_exceptions: Union[Type[Exception], Tuple[Type[Exception], ...]],
         max_tries: Optional[int],
         max_time: Optional[float],
@@ -169,8 +176,9 @@ class RetryInvoker:
         on_giveup: Optional[Callable[[RetryState], None]] = None,
         jitter: Optional[Callable[[float], float]] = full_jitter,
         should_giveup: Optional[Callable[[Exception], bool]] = None,
+        wait_function: Optional[Callable[[float], None]] = None,
     ) -> None:
-        self.wait_factory = wait_factory
+        self.wait_gen_factory = wait_gen_factory
         self.recoverable_exceptions = recoverable_exceptions
         self.max_tries = max_tries
         self.max_time = max_time
@@ -179,6 +187,9 @@ class RetryInvoker:
         self.on_giveup = on_giveup
         self.jitter = jitter
         self.should_giveup = should_giveup
+        if wait_function is None:
+            wait_function = time.sleep
+        self.wait_function = wait_function
 
     # pylint: disable-next=too-many-locals
     def invoke(
@@ -212,13 +223,13 @@ class RetryInvoker:
         Raises
         ------
         Exception
-            If the number of tries exceeds `max_tries`, if the total time
-            exceeds `max_time`, if `wait_factory()` generator raises `StopInteration`,
+            If the number of tries exceeds `max_tries`, if the total time exceeds
+            `max_time`, if `wait_gen_factory()` generator raises `StopInteration`,
             or if the `should_giveup` returns True for a raised exception.
 
         Notes
         -----
-        The time between retries is determined by the provided `wait_factory()`
+        The time between retries is determined by the provided `wait_gen_factory()`
         generator and can optionally be jittered using the `jitter` function.
         The recoverable exceptions that trigger a retry, as well as conditions to
         stop retries, are also determined by the class's initialization parameters.
@@ -231,13 +242,13 @@ class RetryInvoker:
                 handler(cast(RetryState, ref_state[0]))
 
         try_cnt = 0
-        wait_generator = self.wait_factory()
-        start = time.time()
+        wait_generator = self.wait_gen_factory()
+        start = time.monotonic()
         ref_state: List[Optional[RetryState]] = [None]
 
         while True:
             try_cnt += 1
-            elapsed_time = time.time() - start
+            elapsed_time = time.monotonic() - start
             state = RetryState(
                 target=target,
                 args=args,
@@ -282,7 +293,7 @@ class RetryInvoker:
                 try_call_event_handler(self.on_backoff)
 
                 # Sleep
-                time.sleep(wait_time)
+                self.wait_function(state.actual_wait)
             else:
                 # Trigger success event
                 try_call_event_handler(self.on_success)

flwr/server/server_app.py

@@ -18,6 +18,7 @@
 from typing import Callable, Optional
 
 from flwr.common import Context, RecordSet
+from flwr.common.logger import warn_preview_feature
 from flwr.server.strategy import Strategy
 
 from .client_manager import ClientManager
@@ -120,6 +121,8 @@ class ServerApp:
                     """,
                 )
 
+            warn_preview_feature("ServerApp-register-main-function")
+
             # Register provided function with the ServerApp object
             self._main = main_fn
 

flwr/server/superlink/fleet/message_handler/message_handler.py

@@ -63,7 +63,8 @@ def ping(
     state: State,  # pylint: disable=unused-argument
 ) -> PingResponse:
     """."""
-    return PingResponse(success=True)
+    res = state.acknowledge_ping(request.node.node_id, request.ping_interval)
+    return PingResponse(success=res)
 
 
 def pull_task_ins(request: PullTaskInsRequest, state: State) -> PullTaskInsResponse:

flwr/server/superlink/fleet/rest_rere/rest_api.py

@@ -21,6 +21,7 @@ from flwr.common.constant import MISSING_EXTRA_REST
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     DeleteNodeRequest,
+    PingRequest,
     PullTaskInsRequest,
     PushTaskResRequest,
 )
@@ -152,11 +153,38 @@ async def push_task_res(request: Request) -> Response:  # Check if token is need
     )
 
 
+async def ping(request: Request) -> Response:
+    """Ping."""
+    _check_headers(request.headers)
+
+    # Get the request body as raw bytes
+    ping_request_bytes: bytes = await request.body()
+
+    # Deserialize ProtoBuf
+    ping_request_proto = PingRequest()
+    ping_request_proto.ParseFromString(ping_request_bytes)
+
+    # Get state from app
+    state: State = app.state.STATE_FACTORY.state()
+
+    # Handle message
+    ping_response_proto = message_handler.ping(request=ping_request_proto, state=state)
+
+    # Return serialized ProtoBuf
+    ping_response_bytes = ping_response_proto.SerializeToString()
+    return Response(
+        status_code=200,
+        content=ping_response_bytes,
+        headers={"Content-Type": "application/protobuf"},
+    )
+
+
 routes = [
     Route("/api/v0/fleet/create-node", create_node, methods=["POST"]),
     Route("/api/v0/fleet/delete-node", delete_node, methods=["POST"]),
     Route("/api/v0/fleet/pull-task-ins", pull_task_ins, methods=["POST"]),
     Route("/api/v0/fleet/push-task-res", push_task_res, methods=["POST"]),
+    Route("/api/v0/fleet/ping", ping, methods=["POST"]),
 ]
 
 app: Starlette = Starlette(

flwr/server/superlink/fleet/vce/vce_api.py

@@ -223,6 +223,7 @@ async def run(
 
 
 # pylint: disable=too-many-arguments,unused-argument,too-many-locals,too-many-branches
+# pylint: disable=too-many-statements
 def start_vce(
     backend_name: str,
     backend_config_json_stream: str,
@@ -341,6 +342,13 @@ def start_vce(
             )
         )
     except LoadClientAppError as loadapp_ex:
+        f_stop_delay = 10
+        log(
+            ERROR,
+            "LoadClientAppError exception encountered. Terminating simulation in %is",
+            f_stop_delay,
+        )
+        time.sleep(f_stop_delay)
         f_stop.set()  # set termination event
         raise loadapp_ex
     except Exception as ex:

{flwr_nightly-1.8.0.dev20240328.dist-info → flwr_nightly-1.8.0.dev20240401.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.8.0.dev20240328
+Version: 1.8.0.dev20240401
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0

{flwr_nightly-1.8.0.dev20240328.dist-info → flwr_nightly-1.8.0.dev20240401.dist-info}/RECORD

@@ -28,14 +28,15 @@ flwr/cli/run/__init__.py,sha256=oCd6HmQDx-sqver1gecgx-uMA38BLTSiiKpl7RGNceg,789
 flwr/cli/run/run.py,sha256=C7Yh-Y0f64PEabb9733jBKIhhOUFpcRmCZJIDtv-NG8,2329
 flwr/cli/utils.py,sha256=_V2BlFVNNG2naZrq227fZ8o4TxBN_hB-4fQsen9uQoo,2300
 flwr/client/__init__.py,sha256=futk_IdY_N1h8BTve4Iru51bxm7H1gv58ZPIXWi5XUA,1187
-flwr/client/app.py,sha256=bRrLIfn3MIZKfsVq8AlgYmhpI08UTioaOlyx6YLaLR4,25536
+flwr/client/app.py,sha256=3k1qi_OnN17F7Lwz4zmU6O9Ijsvy9UtfCBjy0qZ3EoY,25445
 flwr/client/client.py,sha256=Vp9UkOkoHdNfn6iMYZsj_5m_GICiFfUlKEVaLad-YhM,8183
-flwr/client/client_app.py,sha256=30Tl_AOEi4CE8wEQbKJ3tWg4GfbsSoV1Ztc8iWE0ge4,8047
+flwr/client/client_app.py,sha256=jqZGliZH7unwJ6y1lQOpGjrMvuOt0P5plVveaPeJBXw,8315
 flwr/client/dpfedavg_numpy_client.py,sha256=9Tnig4iml2J88HBKNahegjXjbfvIQyBtaIQaqjbeqsA,7435
 flwr/client/grpc_client/__init__.py,sha256=LsnbqXiJhgQcB0XzAlUQgPx011Uf7Y7yabIC1HxivJ8,735
 flwr/client/grpc_client/connection.py,sha256=w3Lble9-eCzNOR7fBUsVedVCK4ui9QPhK7i7Ew_a5Vk,8717
 flwr/client/grpc_rere_client/__init__.py,sha256=avn6W_vHEM_yZEB1S7hCZgnTbXb6ZujqRP_vAzyXu-0,752
-flwr/client/grpc_rere_client/connection.py,sha256=3kpnUbS06rNQ969EybGx7zZfQPc2JmCuViyrIt610V0,7421
+flwr/client/grpc_rere_client/connection.py,sha256=boYRkyesnpdUWQL-DSA660tINCDGxRIPdXT8hc9uHfo,8525
+flwr/client/heartbeat.py,sha256=6Ix2Du9SDlXU_nre48WIDUXDy3AVoZsGKacSq2NqT5c,2377
 flwr/client/message_handler/__init__.py,sha256=abHvBRJJiiaAMNgeILQbMOa6h8WqMK2BcnvxwQZFpic,719
 flwr/client/message_handler/message_handler.py,sha256=ml_FlduAJ5pxO31n1tKRrWfQRSxkMgKLbwXXcRsNSos,6553
 flwr/client/message_handler/task_handler.py,sha256=ZDJBKmrn2grRMNl1rU1iGs7FiMHL5VmZiSp_6h9GHVU,1824
@@ -51,11 +52,11 @@ flwr/client/node_state.py,sha256=KTTs_l4I0jBM7IsSsbAGjhfL_yZC3QANbzyvyfZBRDM,177
 flwr/client/node_state_tests.py,sha256=gPwz0zf2iuDSa11jedkur_u3Xm7lokIDG5ALD2MCvSw,2195
 flwr/client/numpy_client.py,sha256=u76GWAdHmJM88Agm2EgLQSvO8Jnk225mJTk-_TmPjFE,10283
 flwr/client/rest_client/__init__.py,sha256=ThwOnkMdzxo_UuyTI47Q7y9oSpuTgNT2OuFvJCfuDiw,735
-flwr/client/rest_client/connection.py,sha256=WGkml4gmrbbJ6OAmW3jARmL5nGaOoXo7JjWQbhPhizM,12422
+flwr/client/rest_client/connection.py,sha256=FdK0hQUFBX4jY68wnlRlFJEKADD81fFT8ubhBI0Mm78,14447
 flwr/client/typing.py,sha256=c9EvjlEjasxn1Wqx6bGl6Xg6vM1gMFfmXht-E2i5J-k,1006
 flwr/common/__init__.py,sha256=dHOptgKxna78CEQLD5Yu0QIsoSgpIIw5AhIUZCHDWAU,3721
 flwr/common/address.py,sha256=iTAN9jtmIGMrWFnx9XZQl45ZEtQJVZZLYPRBSNVARGI,1882
-flwr/common/constant.py,sha256=vd4d9VRaft3j6hc3MCWuB4zu2_BVjbVOUfu7kLBy0_g,1954
+flwr/common/constant.py,sha256=brSqgGzA21eDqL8WxYOb-MTYcnsAkT4g0wseyRPE7rI,2084
 flwr/common/context.py,sha256=ounF-mWPPtXGwtae3sg5EhF58ScviOa3MVqxRpGVu-8,1313
 flwr/common/date.py,sha256=UWhBZj49yX9LD4BmatS_ZFZu_-kweGh0KQJ1djyWWH4,891
 flwr/common/differential_privacy.py,sha256=WZWrL7C9XaB9l9NDkLDI5PvM7jwcoTTFu08ZVG8-M5Q,6113
@@ -63,7 +64,7 @@ flwr/common/differential_privacy_constants.py,sha256=c7b7tqgvT7yMK0XN9ndiTBs4mQf
 flwr/common/dp.py,sha256=Hc3lLHihjexbJaD_ft31gdv9XRcwOTgDBwJzICuok3A,2004
 flwr/common/exit_handlers.py,sha256=2Nt0wLhc17KQQsLPFSRAjjhUiEFfJK6tNozdGiIY4Fs,2812
 flwr/common/grpc.py,sha256=HimjpTtIY3Vfqtlq3u-CYWjqAl9rSn0uo3A8JjhUmwQ,2273
-flwr/common/logger.py,sha256=Plhm9fsi4ewb90eGALQZ9xBkR0cGEsckX5RLSMEaS3M,6118
+flwr/common/logger.py,sha256=3hfKun9YISWj4i_QhxgZdnaHJc4x-QvFJQJTKHZ2KHs,6096
 flwr/common/message.py,sha256=vgFSCOkPbl60iS4-XQJ8-rHL54MvNc2AwNSSxVl6qYY,11773
 flwr/common/object_ref.py,sha256=ELoUCAFO-vbjJC41CGpa-WBG2SLYe3ErW-d9YCG3zqA,4961
 flwr/common/parameter.py,sha256=-bFAUayToYDF50FZGrBC1hQYJCQDtB2bbr3ZuVLMtdE,2095
@@ -76,7 +77,7 @@ flwr/common/record/parametersrecord.py,sha256=WSqtRrYvI-mRzkEwv5s-EG-yE5uizJ8zy9
 flwr/common/record/recordset.py,sha256=o5UwLubotz1KE9HCoEIP5kK0f0dlIzpFpS1xeQvxo08,3016
 flwr/common/record/typeddict.py,sha256=2NW8JF27p1uNWaqDbJ7bMkItA5x4ygYT8aHrf8NaqnE,3879
 flwr/common/recordset_compat.py,sha256=BjxeuvlCaP94yIiKOyFFTRBUH_lprFWSLo8U8q3BDbs,13798
-flwr/common/retry_invoker.py,sha256=H_hKqKaEI8vZPywWmoAtJYkcUnKhlYc4kV63zRY0kWA,10856
+flwr/common/retry_invoker.py,sha256=6zpjE5TXw-AuPz6Q4geVW8IU6S8sGaGaehLP3HTmlMw,11669
 flwr/common/secure_aggregation/__init__.py,sha256=29nHIUO2L8-KhNHQ2KmIgRo_4CPkq4LgLCUN0on5FgI,731
 flwr/common/secure_aggregation/crypto/__init__.py,sha256=dz7pVx2aPrHxr_AwgO5mIiTzu4PcvUxRq9NLBbFcsf8,738
 flwr/common/secure_aggregation/crypto/shamir.py,sha256=yY35ZgHlB4YyGW_buG-1X-0M-ejXuQzISgYLgC_Z9TY,2792
@@ -135,7 +136,7 @@ flwr/server/driver/grpc_driver.py,sha256=D2n3_Es_DHFgQsq_TjYVEz8RYJJJYoe24E1voza
 flwr/server/history.py,sha256=hDsoBaA4kUa6d1yvDVXuLluBqOBKSm0_fVDtUtYJkmg,5121
 flwr/server/run_serverapp.py,sha256=3hoXa57T4L1vOWVWPSSdZ_UyRO-uTwUIrhha6TJAXMg,5592
 flwr/server/server.py,sha256=UnBRlI6AGTj0nKeRtEQ3IalM3TJmggMKXhDyn8yKZNk,17664
-flwr/server/server_app.py,sha256=lYUzvzgoPSkOB_6ZxrLcpnPixKQY8Uq5XVD8Mb1Cino,4280
+flwr/server/server_app.py,sha256=KgAT_HqsfseTLNnfX2ph42PBbVqQ0lFzvYrT90V34y0,4402
 flwr/server/server_config.py,sha256=CZaHVAsMvGLjpWVcLPkiYxgJN4xfIyAiUrCI3fETKY4,1349
 flwr/server/strategy/__init__.py,sha256=7eVZ3hQEg2BgA_usAeL6tsLp9T6XI1VYYoFy08Xn-ew,2836
 flwr/server/strategy/aggregate.py,sha256=QyRIJtI5gnuY1NbgrcrOvkHxGIxBvApq7d9Y4xl-6W4,13468
@@ -174,14 +175,14 @@ flwr/server/superlink/fleet/grpc_bidi/grpc_server.py,sha256=1QyBX5qcFPjMVlv7Trvn
 flwr/server/superlink/fleet/grpc_rere/__init__.py,sha256=bEJOMWbSlqkw-y5ZHtEXczhoSlAxErcRYffmTMQAV8M,758
 flwr/server/superlink/fleet/grpc_rere/fleet_servicer.py,sha256=LC_ntiLZMIZkspwjtQ9_MZ4agzArebO4HIVJ3YOrFx8,3036
 flwr/server/superlink/fleet/message_handler/__init__.py,sha256=hEY0l61ojH8Iz30_K1btm1HJ6J49iZJSFUsVYqUTw3A,731
-flwr/server/superlink/fleet/message_handler/message_handler.py,sha256=N5VqopkVuXWg0h1_qzINUQ59bFKq-CXbKSi5DmSSrxs,3161
+flwr/server/superlink/fleet/message_handler/message_handler.py,sha256=jz8l9xZX_ekv_TUv7pdYsByUWbvnLj1PfsOdct1gZ38,3238
 flwr/server/superlink/fleet/rest_rere/__init__.py,sha256=VKDvDq5H8koOUztpmQacVzGJXPLEEkL1Vmolxt3mvnY,735
-flwr/server/superlink/fleet/rest_rere/rest_api.py,sha256=7JCs7NW4Qq8W5QhXxqsQNFiCLlRY-b_iD420vH1Mu-U,5906
+flwr/server/superlink/fleet/rest_rere/rest_api.py,sha256=_tGtARm4x957Fu1EWoDieqOzV9CQZTM4GgKe2GxIOvw,6734
 flwr/server/superlink/fleet/vce/__init__.py,sha256=36MHKiefnJeyjwMQzVUK4m06Ojon3WDcwZGQsAcyVhQ,783
 flwr/server/superlink/fleet/vce/backend/__init__.py,sha256=oBIzmnrSSRvH_H0vRGEGWhWzQQwqe3zn6e13RsNwlIY,1466
 flwr/server/superlink/fleet/vce/backend/backend.py,sha256=LJsKl7oixVvptcG98Rd9ejJycNWcEVB0ODvSreLGp-A,2260
 flwr/server/superlink/fleet/vce/backend/raybackend.py,sha256=TaT2EpbVEsIY0EDzF8obadyZaSXjD38TFGdDPI-ytD0,6375
-flwr/server/superlink/fleet/vce/vce_api.py,sha256=EV4ISvHZPucVLD3lYVFF5fQ4yyxmoaoZMdp_1B2k6J8,11789
+flwr/server/superlink/fleet/vce/vce_api.py,sha256=8zfJf2_BtvxZHI0-Konpa6sTg9l4KXOLgCIkt38C6CE,12041
 flwr/server/superlink/state/__init__.py,sha256=ij-7Ms-hyordQdRmGQxY1-nVa4OhixJ0jr7_YDkys0s,1003
 flwr/server/superlink/state/in_memory_state.py,sha256=XfdCGRzFut9xlf7AsDAhhAmBw-nKDBjmPWAI--espj0,8707
 flwr/server/superlink/state/sqlite_state.py,sha256=1SR6Zz6ud0tSSx940gTfa0vct_GH2n0cX_vnhoAEMlQ,22005
@@ -204,8 +205,8 @@ flwr/simulation/ray_transport/ray_actor.py,sha256=OWjgYW--fswkEDqTP9L_cZblBeUVL5
 flwr/simulation/ray_transport/ray_client_proxy.py,sha256=oDu4sEPIOu39vrNi-fqDAe10xtNUXMO49bM2RWfRcyw,6738
 flwr/simulation/ray_transport/utils.py,sha256=TYdtfg1P9VfTdLMOJlifInGpxWHYs9UfUqIv2wfkRLA,2392
 flwr/simulation/run_simulation.py,sha256=HiIH6aa_v56NfKQN5ZBd94NyVfaZNyFs43_kItYsQXU,15685
-flwr_nightly-1.8.0.dev20240328.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-flwr_nightly-1.8.0.dev20240328.dist-info/METADATA,sha256=8FreyB01iNsm1LCt2zh74ZE46oAFcKAieUIrZQEtTgI,15257
-flwr_nightly-1.8.0.dev20240328.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-flwr_nightly-1.8.0.dev20240328.dist-info/entry_points.txt,sha256=utu2wybGyYJSTtsB2ktY_gmy-XtMFo9EFZdishX0zR4,320
-flwr_nightly-1.8.0.dev20240328.dist-info/RECORD,,
+flwr_nightly-1.8.0.dev20240401.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+flwr_nightly-1.8.0.dev20240401.dist-info/METADATA,sha256=9EApt2x0ajI0-A4ms8brLZ45cWsrgTPSmrKs6X1maPs,15257
+flwr_nightly-1.8.0.dev20240401.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+flwr_nightly-1.8.0.dev20240401.dist-info/entry_points.txt,sha256=utu2wybGyYJSTtsB2ktY_gmy-XtMFo9EFZdishX0zR4,320
+flwr_nightly-1.8.0.dev20240401.dist-info/RECORD,,