flwr 1.20.0__py3-none-any.whl → 1.21.0__py3-none-any.whl

flwr/supernode/runtime/run_clientapp.py

@@ -16,9 +16,6 @@
 
 
 import gc
-import os
-import threading
-import time
 from logging import DEBUG, ERROR, INFO
 from typing import Optional
 
@@ -63,23 +60,15 @@ from flwr.proto.appio_pb2 import (  # pylint: disable=E0611
     PushAppOutputsRequest,
     PushAppOutputsResponse,
 )
-
-# pylint: disable=E0611
-from flwr.proto.clientappio_pb2 import (
-    GetRunIdsWithPendingMessagesRequest,
-    GetRunIdsWithPendingMessagesResponse,
-    RequestTokenRequest,
-    RequestTokenResponse,
-)
 from flwr.proto.clientappio_pb2_grpc import ClientAppIoStub
 from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
+from flwr.supercore.app_utils import start_parent_process_monitor
 from flwr.supercore.utils import mask_string
 
 
 def run_clientapp(  # pylint: disable=R0913, R0914, R0917
     clientappio_api_address: str,
-    run_once: bool,
-    token: Optional[str] = None,
+    token: str,
     flwr_dir: Optional[str] = None,
     certificates: Optional[bytes] = None,
     parent_pid: Optional[int] = None,
@@ -102,113 +91,61 @@ def run_clientapp(  # pylint: disable=R0913, R0914, R0917
         stub = ClientAppIoStub(channel)
         _wrap_stub(stub, _make_simple_grpc_retry_invoker())
 
-        while True:
-            # If token is not set, loop until token is received from SuperNode
-            if token is None:
-                token = get_token(stub)
+        # Pull Message, Context, Run and (optional) FAB from SuperNode
+        message, context, run, fab = pull_clientappinputs(stub=stub, token=token)
 
-            # Pull Message, Context, Run and (optional) FAB from SuperNode
-            message, context, run, fab = pull_clientappinputs(stub=stub, token=token)
+        # Install FAB, if provided
+        if fab:
+            log(DEBUG, "[flwr-clientapp] Start FAB installation.")
+            install_from_fab(fab.content, flwr_dir=flwr_dir_, skip_prompt=True)
 
-            # Install FAB, if provided
-            if fab:
-                log(DEBUG, "[flwr-clientapp] Start FAB installation.")
-                install_from_fab(fab.content, flwr_dir=flwr_dir_, skip_prompt=True)
+        load_client_app_fn = get_load_client_app_fn(
+            default_app_ref="",
+            app_path=None,
+            multi_app=True,
+            flwr_dir=str(flwr_dir_),
+        )
 
-            load_client_app_fn = get_load_client_app_fn(
-                default_app_ref="",
-                app_path=None,
-                multi_app=True,
-                flwr_dir=str(flwr_dir_),
+        try:
+            # Load ClientApp
+            log(DEBUG, "[flwr-clientapp] Start `ClientApp` Loading.")
+            client_app: ClientApp = load_client_app_fn(
+                run.fab_id, run.fab_version, fab.hash_str if fab else ""
             )
 
-            try:
-                # Load ClientApp
-                log(DEBUG, "[flwr-clientapp] Start `ClientApp` Loading.")
-                client_app: ClientApp = load_client_app_fn(
-                    run.fab_id, run.fab_version, fab.hash_str if fab else ""
-                )
-
-                # Execute ClientApp
-                reply_message = client_app(message=message, context=context)
+            # Execute ClientApp
+            reply_message = client_app(message=message, context=context)
 
-            except Exception as ex:  # pylint: disable=broad-exception-caught
-                # Don't update/change NodeState
+        except Exception as ex:  # pylint: disable=broad-exception-caught
+            # Don't update/change NodeState
 
-                e_code = ErrorCode.CLIENT_APP_RAISED_EXCEPTION
-                # Ex fmt: "<class 'ZeroDivisionError'>:<'division by zero'>"
-                reason = str(type(ex)) + ":<'" + str(ex) + "'>"
-                exc_entity = "ClientApp"
-                if isinstance(ex, LoadClientAppError):
-                    reason = (
-                        "An exception was raised when attempting to load `ClientApp`"
-                    )
-                    e_code = ErrorCode.LOAD_CLIENT_APP_EXCEPTION
+            e_code = ErrorCode.CLIENT_APP_RAISED_EXCEPTION
+            # Ex fmt: "<class 'ZeroDivisionError'>:<'division by zero'>"
+            reason = str(type(ex)) + ":<'" + str(ex) + "'>"
+            exc_entity = "ClientApp"
+            if isinstance(ex, LoadClientAppError):
+                reason = "An exception was raised when attempting to load `ClientApp`"
+                e_code = ErrorCode.LOAD_CLIENT_APP_EXCEPTION
 
-                log(ERROR, "%s raised an exception", exc_entity, exc_info=ex)
+            log(ERROR, "%s raised an exception", exc_entity, exc_info=ex)
 
-                # Create error message
-                reply_message = Message(
-                    Error(code=e_code, reason=reason), reply_to=message
-                )
+            # Create error message
+            reply_message = Message(Error(code=e_code, reason=reason), reply_to=message)
 
-            # Push Message and Context to SuperNode
-            _ = push_clientappoutputs(
-                stub=stub, token=token, message=reply_message, context=context
-            )
-
-            del client_app, message, context, run, fab, reply_message
-            gc.collect()
-
-            # Reset token to `None` to prevent flwr-clientapp from trying to pull the
-            # same inputs again
-            token = None
+        # Push Message and Context to SuperNode
+        _ = push_clientappoutputs(
+            stub=stub, token=token, message=reply_message, context=context
+        )
 
-            # Stop the loop if `flwr-clientapp` is expected to process only a single
-            # message
-            if run_once:
-                break
+        del client_app, message, context, run, fab, reply_message
+        gc.collect()
 
-    except KeyboardInterrupt:
-        log(INFO, "Closing connection")
     except grpc.RpcError as e:
         log(ERROR, "GRPC error occurred: %s", str(e))
     finally:
         channel.close()
 
 
-def start_parent_process_monitor(
-    parent_pid: int,
-) -> None:
-    """Monitor the parent process and exit if it terminates."""
-
-    def monitor() -> None:
-        while True:
-            time.sleep(0.2)
-            if os.getppid() != parent_pid:
-                os.kill(os.getpid(), 9)
-
-    threading.Thread(target=monitor, daemon=True).start()
-
-
-def get_token(stub: ClientAppIoStub) -> str:
-    """Get a token from SuperNode."""
-    log(DEBUG, "[flwr-clientapp] Request token")
-    while True:
-        res: GetRunIdsWithPendingMessagesResponse = stub.GetRunIdsWithPendingMessages(
-            GetRunIdsWithPendingMessagesRequest()
-        )
-
-        for run_id in res.run_ids:
-            tk_res: RequestTokenResponse = stub.RequestToken(
-                RequestTokenRequest(run_id=run_id)
-            )
-            if tk_res.token:
-                return tk_res.token
-
-        time.sleep(1)  # Wait before retrying to get run IDs
-
-
 def pull_clientappinputs(
     stub: ClientAppIoStub, token: str
 ) -> tuple[Message, Context, Run, Optional[Fab]]:

flwr/supernode/servicer/clientappio/clientappio_servicer.py

@@ -36,6 +36,8 @@ from flwr.common.typing import Fab, Run
 # pylint: disable=E0611
 from flwr.proto import clientappio_pb2_grpc
 from flwr.proto.appio_pb2 import (  # pylint: disable=E0401
+    ListAppsToLaunchRequest,
+    ListAppsToLaunchResponse,
     PullAppInputsRequest,
     PullAppInputsResponse,
     PullAppMessagesRequest,
@@ -44,10 +46,6 @@ from flwr.proto.appio_pb2 import (  # pylint: disable=E0401
     PushAppMessagesResponse,
     PushAppOutputsRequest,
     PushAppOutputsResponse,
-)
-from flwr.proto.clientappio_pb2 import (  # pylint: disable=E0401
-    GetRunIdsWithPendingMessagesRequest,
-    GetRunIdsWithPendingMessagesResponse,
     RequestTokenRequest,
     RequestTokenResponse,
 )
@@ -82,13 +80,13 @@ class ClientAppIoServicer(clientappio_pb2_grpc.ClientAppIoServicer):
         self.ffs_factory = ffs_factory
         self.objectstore_factory = objectstore_factory
 
-    def GetRunIdsWithPendingMessages(
+    def ListAppsToLaunch(
         self,
-        request: GetRunIdsWithPendingMessagesRequest,
+        request: ListAppsToLaunchRequest,
         context: grpc.ServicerContext,
-    ) -> GetRunIdsWithPendingMessagesResponse:
-        """Get run IDs with pending messages."""
-        log(DEBUG, "ClientAppIo.GetRunIdsWithPendingMessages")
+    ) -> ListAppsToLaunchResponse:
+        """Get run IDs with apps to launch."""
+        log(DEBUG, "ClientAppIo.ListAppsToLaunch")
 
         # Initialize state connection
         state = self.state_factory.state()
@@ -97,7 +95,7 @@ class ClientAppIoServicer(clientappio_pb2_grpc.ClientAppIoServicer):
         run_ids = state.get_run_ids_with_pending_messages()
 
         # Return run IDs
-        return GetRunIdsWithPendingMessagesResponse(run_ids=run_ids)
+        return ListAppsToLaunchResponse(run_ids=run_ids)
 
     def RequestToken(
         self, request: RequestTokenRequest, context: grpc.ServicerContext
@@ -109,15 +107,10 @@ class ClientAppIoServicer(clientappio_pb2_grpc.ClientAppIoServicer):
         state = self.state_factory.state()
 
         # Attempt to create a token for the provided run ID
-        try:
-            token = state.create_token(request.run_id)
-        except ValueError:
-            # Return an empty token if A token already exists for this run ID,
-            # indicating the run is in progress
-            return RequestTokenResponse(token="")
+        token = state.create_token(request.run_id)
 
         # Return the token
-        return RequestTokenResponse(token=token)
+        return RequestTokenResponse(token=token or "")
 
     def GetRun(
         self, request: GetRunRequest, context: grpc.ServicerContext

flwr/supernode/start_client_internal.py

@@ -21,7 +21,7 @@ import time
 from collections.abc import Iterator
 from contextlib import contextmanager
 from functools import partial
-from logging import INFO, WARN
+from logging import INFO
 from pathlib import Path
 from typing import Callable, Optional, Union, cast
 
@@ -35,18 +35,15 @@ from flwr.common import GRPC_MAX_MESSAGE_LENGTH, Context, Message, RecordDict
 from flwr.common.address import parse_address
 from flwr.common.config import get_flwr_dir, get_fused_config_from_fab
 from flwr.common.constant import (
-    CLIENT_OCTET,
     CLIENTAPPIO_API_DEFAULT_SERVER_ADDRESS,
     ISOLATION_MODE_SUBPROCESS,
-    MAX_RETRY_DELAY,
-    SERVER_OCTET,
     TRANSPORT_TYPE_GRPC_ADAPTER,
     TRANSPORT_TYPE_GRPC_RERE,
     TRANSPORT_TYPE_REST,
     TRANSPORT_TYPES,
+    ExecPluginType,
 )
-from flwr.common.exit import ExitCode, flwr_exit
-from flwr.common.exit_handlers import register_exit_handlers
+from flwr.common.exit import ExitCode, flwr_exit, register_signal_handlers
 from flwr.common.grpc import generic_create_grpc_server
 from flwr.common.inflatable import iterate_object_tree
 from flwr.common.inflatable_utils import (
@@ -54,12 +51,13 @@ from flwr.common.inflatable_utils import (
     push_object_contents_from_iterable,
 )
 from flwr.common.logger import log
-from flwr.common.retry_invoker import RetryInvoker, RetryState, exponential
+from flwr.common.retry_invoker import RetryInvoker, _make_simple_grpc_retry_invoker
 from flwr.common.telemetry import EventType
 from flwr.common.typing import Fab, Run, RunNotRunningException, UserConfig
 from flwr.proto.clientappio_pb2_grpc import add_ClientAppIoServicer_to_server
 from flwr.proto.message_pb2 import ObjectTree  # pylint: disable=E0611
 from flwr.supercore.ffs import Ffs, FfsFactory
+from flwr.supercore.grpc_health import run_health_server_grpc_no_tls
 from flwr.supercore.object_store import ObjectStore, ObjectStoreFactory
 from flwr.supernode.nodestate import NodeState, NodeStateFactory
 from flwr.supernode.servicer.clientappio import ClientAppIoServicer
@@ -87,6 +85,7 @@ def start_client_internal(
     flwr_path: Optional[Path] = None,
     isolation: str = ISOLATION_MODE_SUBPROCESS,
     clientappio_api_address: str = CLIENTAPPIO_API_DEFAULT_SERVER_ADDRESS,
+    health_server_address: Optional[str] = None,
 ) -> None:
     """Start a Flower client node which connects to a Flower server.
 
@@ -135,6 +134,9 @@ def start_client_internal(
     clientappio_api_address : str
         (default: `CLIENTAPPIO_API_DEFAULT_SERVER_ADDRESS`)
         The SuperNode gRPC server address.
+    health_server_address : Optional[str] (default: None)
+        The address of the health server. If `None` is provided, the health server will
+        NOT be started.
     """
     if insecure is None:
         insecure = root_certificates is None
@@ -145,6 +147,7 @@ def start_client_internal(
     object_store_factory = ObjectStoreFactory()
 
     # Launch ClientAppIo API server
+    grpc_servers = []
     clientappio_server = run_clientappio_api_grpc(
         address=clientappio_api_address,
         state_factory=state_factory,
@@ -152,12 +155,18 @@ def start_client_internal(
         objectstore_factory=object_store_factory,
         certificates=None,
     )
+    grpc_servers.append(clientappio_server)
+
+    # Launch gRPC health server
+    if health_server_address is not None:
+        health_server = run_health_server_grpc_no_tls(health_server_address)
+        grpc_servers.append(health_server)
 
     # Register handlers for graceful shutdown
-    register_exit_handlers(
+    register_signal_handlers(
         event_type=EventType.RUN_SUPERNODE_LEAVE,
         exit_message="SuperNode terminated gracefully.",
-        grpc_servers=[clientappio_server],
+        grpc_servers=grpc_servers,
     )
 
     # Initialize NodeState, Ffs, and ObjectStore
@@ -165,6 +174,15 @@ def start_client_internal(
     ffs = ffs_factory.ffs()
     store = object_store_factory.store()
 
+    # Launch the SuperExec if the isolation mode is `subprocess`
+    if isolation == ISOLATION_MODE_SUBPROCESS:
+        command = ["flower-superexec", "--insecure"]
+        command += ["--appio-api-address", clientappio_api_address]
+        command += ["--plugin-type", ExecPluginType.CLIENT_APP]
+        command += ["--parent-pid", str(os.getpid())]
+        # pylint: disable-next=consider-using-with
+        subprocess.Popen(command)
+
     with _init_connection(
         transport=transport,
         server_address=server_address,
@@ -208,35 +226,6 @@ def start_client_internal(
                 confirm_message_received=confirm_message_received,
             )
 
-            # Two isolation modes:
-            # 1. `subprocess`: SuperNode is starting the ClientApp
-            #    process as a subprocess.
-            # 2. `process`: ClientApp process gets started separately
-            #    (via `flwr-clientapp`), for example, in a separate
-            #    Docker container.
-
-            # Mode 1: SuperNode starts ClientApp as subprocess
-            start_subprocess = isolation == ISOLATION_MODE_SUBPROCESS
-
-            if start_subprocess and run_id is not None:
-                _octet, _colon, _port = clientappio_api_address.rpartition(":")
-                io_address = (
-                    f"{CLIENT_OCTET}:{_port}"
-                    if _octet == SERVER_OCTET
-                    else clientappio_api_address
-                )
-                # Start ClientApp subprocess
-                command = [
-                    "flwr-clientapp",
-                    "--clientappio-api-address",
-                    io_address,
-                    "--parent-pid",
-                    str(os.getpid()),
-                    "--insecure",
-                    "--run-once",
-                ]
-                subprocess.run(command, check=False)
-
             # No message has been pulled therefore we can skip the push stage.
             if run_id is None:
                 # If no message was received, wait for a while
@@ -521,44 +510,14 @@ def _make_fleet_connection_retry_invoker(
     connection_error_type: type[Exception] = RpcError,
 ) -> RetryInvoker:
     """Create a retry invoker for fleet connection."""
+    retry_invoker = _make_simple_grpc_retry_invoker()
+    retry_invoker.recoverable_exceptions = connection_error_type
+    if max_retries is not None:
+        retry_invoker.max_tries = max_retries + 1
+    if max_wait_time is not None:
+        retry_invoker.max_time = max_wait_time
 
-    def _on_success(retry_state: RetryState) -> None:
-        if retry_state.tries > 1:
-            log(
-                INFO,
-                "Connection successful after %.2f seconds and %s tries.",
-                retry_state.elapsed_time,
-                retry_state.tries,
-            )
-
-    def _on_backoff(retry_state: RetryState) -> None:
-        if retry_state.tries == 1:
-            log(WARN, "Connection attempt failed, retrying...")
-        else:
-            log(
-                WARN,
-                "Connection attempt failed, retrying in %.2f seconds",
-                retry_state.actual_wait,
-            )
-
-    return RetryInvoker(
-        wait_gen_factory=lambda: exponential(max_delay=MAX_RETRY_DELAY),
-        recoverable_exceptions=connection_error_type,
-        max_tries=max_retries + 1 if max_retries is not None else None,
-        max_time=max_wait_time,
-        on_giveup=lambda retry_state: (
-            log(
-                WARN,
-                "Giving up reconnection after %.2f seconds and %s tries.",
-                retry_state.elapsed_time,
-                retry_state.tries,
-            )
-            if retry_state.tries > 1
-            else None
-        ),
-        on_success=_on_success,
-        on_backoff=_on_backoff,
-    )
+    return retry_invoker
 
 
 def run_clientappio_api_grpc(
@@ -584,6 +543,6 @@ def run_clientappio_api_grpc(
         max_message_length=GRPC_MAX_MESSAGE_LENGTH,
         certificates=certificates,
     )
-    log(INFO, "Starting Flower ClientAppIo gRPC server on %s", address)
+    log(INFO, "Flower Deployment Runtime: Starting ClientAppIo API on %s", address)
     clientappio_grpc_server.start()
     return clientappio_grpc_server

{flwr-1.20.0.dist-info → flwr-1.21.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: flwr
-Version: 1.20.0
+Version: 1.21.0
 Summary: Flower: A Friendly Federated AI Framework
 License: Apache-2.0
 Keywords: Artificial Intelligence,Federated AI,Federated Analytics,Federated Evaluation,Federated Learning,Flower,Machine Learning
@@ -172,8 +172,9 @@ Flower Baselines is a collection of community-contributed projects that reproduc
 - [FedOpt](https://github.com/adap/flower/tree/main/baselines/flwr_baselines/flwr_baselines/publications/adaptive_federated_optimization)
 
 Please refer to the [Flower Baselines Documentation](https://flower.ai/docs/baselines/) for a detailed categorization of baselines and for additional info including:
-* [How to use Flower Baselines](https://flower.ai/docs/baselines/how-to-use-baselines.html)
-* [How to contribute a new Flower Baseline](https://flower.ai/docs/baselines/how-to-contribute-baselines.html)
+
+- [How to use Flower Baselines](https://flower.ai/docs/baselines/how-to-use-baselines.html)
+- [How to contribute a new Flower Baseline](https://flower.ai/docs/baselines/how-to-contribute-baselines.html)
 
 ## Flower Usage Examples