flwr-nightly 1.13.0.dev20241111__py3-none-any.whl → 1.14.0.dev20241126__py3-none-any.whl

flwr/server/driver/grpc_driver.py

@@ -17,20 +17,17 @@
 import time
 import warnings
 from collections.abc import Iterable
-from logging import DEBUG, WARNING
-from typing import Optional, cast
+from logging import DEBUG, INFO, WARN, WARNING
+from typing import Any, Optional, cast
 
 import grpc
 
 from flwr.common import DEFAULT_TTL, Message, Metadata, RecordSet
-from flwr.common.constant import SERVERAPPIO_API_DEFAULT_ADDRESS
+from flwr.common.constant import MAX_RETRY_DELAY, SERVERAPPIO_API_DEFAULT_CLIENT_ADDRESS
 from flwr.common.grpc import create_channel
 from flwr.common.logger import log
-from flwr.common.serde import (
-    message_from_taskres,
-    message_to_taskins,
-    user_config_from_proto,
-)
+from flwr.common.retry_invoker import RetryInvoker, RetryState, exponential
+from flwr.common.serde import message_from_taskres, message_to_taskins, run_from_proto
 from flwr.common.typing import Run
 from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
 from flwr.proto.run_pb2 import GetRunRequest, GetRunResponse  # pylint: disable=E0611
@@ -70,7 +67,7 @@ class GrpcDriver(Driver):
 
     def __init__(  # pylint: disable=too-many-arguments
         self,
-        serverappio_service_address: str = SERVERAPPIO_API_DEFAULT_ADDRESS,
+        serverappio_service_address: str = SERVERAPPIO_API_DEFAULT_CLIENT_ADDRESS,
         root_certificates: Optional[bytes] = None,
     ) -> None:
         self._addr = serverappio_service_address
@@ -79,6 +76,7 @@ class GrpcDriver(Driver):
         self._grpc_stub: Optional[ServerAppIoStub] = None
         self._channel: Optional[grpc.Channel] = None
         self.node = Node(node_id=0, anonymous=True)
+        self._retry_invoker = _make_simple_grpc_retry_invoker()
 
     @property
     def _is_connected(self) -> bool:
@@ -99,6 +97,7 @@ class GrpcDriver(Driver):
             root_certificates=self._cert,
         )
         self._grpc_stub = ServerAppIoStub(self._channel)
+        _wrap_stub(self._grpc_stub, self._retry_invoker)
         log(DEBUG, "[Driver] Connected to %s", self._addr)
 
     def _disconnect(self) -> None:
@@ -119,13 +118,7 @@ class GrpcDriver(Driver):
         res: GetRunResponse = self._stub.GetRun(req)
         if not res.HasField("run"):
             raise RuntimeError(f"Cannot find the run with ID: {run_id}")
-        self._run = Run(
-            run_id=res.run.run_id,
-            fab_id=res.run.fab_id,
-            fab_version=res.run.fab_version,
-            fab_hash=res.run.fab_hash,
-            override_config=user_config_from_proto(res.run.override_config),
-        )
+        self._run = run_from_proto(res.run)
 
     @property
     def run(self) -> Run:
@@ -265,3 +258,60 @@ class GrpcDriver(Driver):
             return
         # Disconnect
         self._disconnect()
+
+
+def _make_simple_grpc_retry_invoker() -> RetryInvoker:
+    """Create a simple gRPC retry invoker."""
+
+    def _on_sucess(retry_state: RetryState) -> None:
+        if retry_state.tries > 1:
+            log(
+                INFO,
+                "Connection successful after %.2f seconds and %s tries.",
+                retry_state.elapsed_time,
+                retry_state.tries,
+            )
+
+    def _on_backoff(retry_state: RetryState) -> None:
+        if retry_state.tries == 1:
+            log(WARN, "Connection attempt failed, retrying...")
+        else:
+            log(
+                WARN,
+                "Connection attempt failed, retrying in %.2f seconds",
+                retry_state.actual_wait,
+            )
+
+    def _on_giveup(retry_state: RetryState) -> None:
+        if retry_state.tries > 1:
+            log(
+                WARN,
+                "Giving up reconnection after %.2f seconds and %s tries.",
+                retry_state.elapsed_time,
+                retry_state.tries,
+            )
+
+    return RetryInvoker(
+        wait_gen_factory=lambda: exponential(max_delay=MAX_RETRY_DELAY),
+        recoverable_exceptions=grpc.RpcError,
+        max_tries=None,
+        max_time=None,
+        on_success=_on_sucess,
+        on_backoff=_on_backoff,
+        on_giveup=_on_giveup,
+        should_giveup=lambda e: e.code() != grpc.StatusCode.UNAVAILABLE,  # type: ignore
+    )
+
+
+def _wrap_stub(stub: ServerAppIoStub, retry_invoker: RetryInvoker) -> None:
+    """Wrap the gRPC stub with a retry invoker."""
+
+    def make_lambda(original_method: Any) -> Any:
+        return lambda *args, **kwargs: retry_invoker.invoke(
+            original_method, *args, **kwargs
+        )
+
+    for method_name in vars(stub):
+        method = getattr(stub, method_name)
+        if callable(method):
+            setattr(stub, method_name, make_lambda(method))

flwr/server/run_serverapp.py

@@ -15,33 +15,15 @@
 """Run ServerApp."""
 
 
-import argparse
 import sys
-from logging import DEBUG, INFO, WARN
-from pathlib import Path
+from logging import DEBUG, ERROR
 from typing import Optional
 
-from flwr.cli.config_utils import get_fab_metadata
-from flwr.cli.install import install_from_fab
-from flwr.common import Context, EventType, RecordSet, event
-from flwr.common.config import (
-    get_flwr_dir,
-    get_fused_config_from_dir,
-    get_metadata_from_config,
-    get_project_config,
-    get_project_dir,
-)
-from flwr.common.constant import SERVERAPPIO_API_DEFAULT_ADDRESS
-from flwr.common.logger import log, update_console_handler, warn_deprecated_feature
+from flwr.common import Context
+from flwr.common.logger import log, warn_unsupported_feature
 from flwr.common.object_ref import load_app
-from flwr.proto.fab_pb2 import GetFabRequest, GetFabResponse  # pylint: disable=E0611
-from flwr.proto.run_pb2 import (  # pylint: disable=E0611
-    CreateRunRequest,
-    CreateRunResponse,
-)
 
 from .driver import Driver
-from .driver.grpc_driver import GrpcDriver
 from .server_app import LoadServerAppError, ServerApp
 
 
@@ -87,221 +69,9 @@ def run(
 # pylint: disable-next=too-many-branches,too-many-statements,too-many-locals
 def run_server_app() -> None:
     """Run Flower server app."""
-    event(EventType.RUN_SERVER_APP_ENTER)
-
-    args = _parse_args_run_server_app().parse_args()
-
-    # Check if the server app reference is passed.
-    # Since Flower 1.11, passing a reference is not allowed.
-    app_path: Optional[str] = args.app
-    # If the provided app_path doesn't exist, and contains a ":",
-    # it is likely to be a server app reference instead of a path.
-    if app_path is not None and not Path(app_path).exists() and ":" in app_path:
-        sys.exit(
-            "It appears you've passed a reference like `server:app`.\n\n"
-            "Note that since version `1.11.0`, `flower-server-app` no longer supports "
-            "passing a reference to a `ServerApp` attribute. Instead, you need to pass "
-            "the path to Flower app via the argument `--app`. This is the path to a "
-            "directory containing a `pyproject.toml`. You can create a valid Flower "
-            "app by executing `flwr new` and following the prompt."
-        )
-
-    if args.server != SERVERAPPIO_API_DEFAULT_ADDRESS:
-        warn = "Passing flag --server is deprecated. Use --superlink instead."
-        warn_deprecated_feature(warn)
-
-        if args.superlink != SERVERAPPIO_API_DEFAULT_ADDRESS:
-            # if `--superlink` also passed, then
-            # warn user that this argument overrides what was passed with `--server`
-            log(
-                WARN,
-                "Both `--server` and `--superlink` were passed. "
-                "`--server` will be ignored. Connecting to the "
-                "SuperLink ServerAppIo API at %s.",
-                args.superlink,
-            )
-        else:
-            args.superlink = args.server
-
-    update_console_handler(
-        level=DEBUG if args.verbose else INFO,
-        timestamps=args.verbose,
-        colored=True,
-    )
-
-    # Obtain certificates
-    if args.insecure:
-        if args.root_certificates is not None:
-            sys.exit(
-                "Conflicting options: The '--insecure' flag disables HTTPS, "
-                "but '--root-certificates' was also specified. Please remove "
-                "the '--root-certificates' option when running in insecure mode, "
-                "or omit '--insecure' to use HTTPS."
-            )
-        log(
-            WARN,
-            "Option `--insecure` was set. "
-            "Starting insecure HTTP client connected to %s.",
-            args.superlink,
-        )
-        root_certificates = None
-    else:
-        # Load the certificates if provided, or load the system certificates
-        cert_path = args.root_certificates
-        if cert_path is None:
-            root_certificates = None
-        else:
-            root_certificates = Path(cert_path).read_bytes()
-        log(
-            DEBUG,
-            "Starting secure HTTPS client connected to %s "
-            "with the following certificates: %s.",
-            args.superlink,
-            cert_path,
-        )
-
-    if not (app_path is None) ^ (args.run_id is None):
-        raise sys.exit(
-            "Please provide either a Flower App path or a Run ID, but not both. "
-            "For more details, use: ``flower-server-app -h``"
-        )
-
-    # Initialize GrpcDriver
-    if app_path is None:
-        # User provided `--run-id`, but not `app_dir`
-        driver = GrpcDriver(
-            serverappio_service_address=args.superlink,
-            root_certificates=root_certificates,
-        )
-        flwr_dir = get_flwr_dir(args.flwr_dir)
-        driver.set_run(args.run_id)
-        run_ = driver.run
-        if not run_.fab_hash:
-            raise ValueError("FAB hash not provided.")
-        fab_req = GetFabRequest(hash_str=run_.fab_hash)
-        # pylint: disable-next=W0212
-        fab_res: GetFabResponse = driver._stub.GetFab(fab_req)
-        if fab_res.fab.hash_str != run_.fab_hash:
-            raise ValueError("FAB hashes don't match.")
-        install_from_fab(fab_res.fab.content, flwr_dir, True)
-        fab_id, fab_version = get_fab_metadata(fab_res.fab.content)
-
-        app_path = str(get_project_dir(fab_id, fab_version, run_.fab_hash, flwr_dir))
-        config = get_project_config(app_path)
-        run_id = run_.run_id
-    else:
-        # User provided `app_dir`, but not `--run-id`
-        # Create run if run_id is not provided
-        driver = GrpcDriver(
-            serverappio_service_address=args.superlink,
-            root_certificates=root_certificates,
-        )
-        # Load config from the project directory
-        config = get_project_config(app_path)
-        fab_version, fab_id = get_metadata_from_config(config)
-
-        # Create run
-        req = CreateRunRequest(fab_id=fab_id, fab_version=fab_version)
-        res: CreateRunResponse = driver._stub.CreateRun(req)  # pylint: disable=W0212
-        # Fetch full `Run` using `run_id`
-        driver.set_run(res.run_id)  # pylint: disable=W0212
-        run_id = res.run_id
-
-    # Obtain server app reference and the run config
-    server_app_attr = config["tool"]["flwr"]["app"]["components"]["serverapp"]
-    server_app_run_config = get_fused_config_from_dir(
-        Path(app_path), driver.run.override_config
+    warn_unsupported_feature(
+        "The command `flower-server-app` is deprecated and no longer in use. "
+        "Use the `flwr-serverapp` exclusively instead."
     )
-
-    log(DEBUG, "Flower will load ServerApp `%s` in %s", server_app_attr, app_path)
-
-    log(
-        DEBUG,
-        "root_certificates: `%s`",
-        root_certificates,
-    )
-
-    # Initialize Context
-    context = Context(
-        run_id=run_id,
-        node_id=0,
-        node_config={},
-        state=RecordSet(),
-        run_config=server_app_run_config,
-    )
-
-    # Run the ServerApp with the Driver
-    run(
-        driver=driver,
-        context=context,
-        server_app_dir=app_path,
-        server_app_attr=server_app_attr,
-    )
-
-    # Clean up
-    driver.close()
-
-    event(EventType.RUN_SERVER_APP_LEAVE)
-
-
-def _parse_args_run_server_app() -> argparse.ArgumentParser:
-    """Parse flower-server-app command line arguments."""
-    parser = argparse.ArgumentParser(
-        description="Start a Flower server app",
-    )
-
-    parser.add_argument(
-        "app",
-        nargs="?",
-        default=None,
-        help="Load and run the `ServerApp` from the specified Flower App path. "
-        "The `pyproject.toml` file must be located in the root of this path.",
-    )
-    parser.add_argument(
-        "--insecure",
-        action="store_true",
-        help="Run the `ServerApp` without HTTPS. By default, the app runs with "
-        "HTTPS enabled. Use this flag only if you understand the risks.",
-    )
-    parser.add_argument(
-        "--verbose",
-        action="store_true",
-        help="Set the logging to `DEBUG`.",
-    )
-    parser.add_argument(
-        "--root-certificates",
-        metavar="ROOT_CERT",
-        type=str,
-        help="Specifies the path to the PEM-encoded root certificate file for "
-        "establishing secure HTTPS connections.",
-    )
-    parser.add_argument(
-        "--server",
-        default=SERVERAPPIO_API_DEFAULT_ADDRESS,
-        help="Server address",
-    )
-    parser.add_argument(
-        "--superlink",
-        default=SERVERAPPIO_API_DEFAULT_ADDRESS,
-        help="SuperLink ServerAppIo API (gRPC-rere) address "
-        "(IPv4, IPv6, or a domain name)",
-    )
-    parser.add_argument(
-        "--run-id",
-        default=None,
-        type=int,
-        help="The identifier of the run.",
-    )
-    parser.add_argument(
-        "--flwr-dir",
-        default=None,
-        help="""The path containing installed Flower Apps.
-    By default, this value is equal to:
-
-        - `$FLWR_HOME/` if `$FLWR_HOME` is defined
-        - `$XDG_DATA_HOME/.flwr/` if `$XDG_DATA_HOME` is defined
-        - `$HOME/.flwr/` in all other cases
-    """,
-    )
-
-    return parser
+    log(ERROR, "`flower-server-app` used.")
+    sys.exit()

flwr/server/serverapp/app.py

@@ -15,6 +15,7 @@
 """Flower ServerApp process."""
 
 import argparse
+import sys
 from logging import DEBUG, ERROR, INFO
 from pathlib import Path
 from queue import Queue
@@ -23,14 +24,18 @@ from typing import Optional
 
 from flwr.cli.config_utils import get_fab_metadata
 from flwr.cli.install import install_from_fab
-from flwr.common.args import add_args_flwr_app_common, try_obtain_certificates
+from flwr.common.args import add_args_flwr_app_common
 from flwr.common.config import (
     get_flwr_dir,
     get_fused_config_from_dir,
     get_project_config,
     get_project_dir,
 )
-from flwr.common.constant import Status, SubStatus
+from flwr.common.constant import (
+    SERVERAPPIO_API_DEFAULT_CLIENT_ADDRESS,
+    Status,
+    SubStatus,
+)
 from flwr.common.logger import (
     log,
     mirror_output_to_queue,
@@ -62,37 +67,29 @@ def flwr_serverapp() -> None:
     log_queue: Queue[Optional[str]] = Queue()
     mirror_output_to_queue(log_queue)
 
-    parser = argparse.ArgumentParser(
-        description="Run a Flower ServerApp",
-    )
-    parser.add_argument(
-        "--superlink",
-        type=str,
-        help="Address of SuperLink's ServerAppIo API",
-    )
-    parser.add_argument(
-        "--run-once",
-        action="store_true",
-        help="When set, this process will start a single ServerApp for a pending Run. "
-        "If there is no pending Run, the process will exit.",
-    )
-    add_args_flwr_app_common(parser=parser)
-    args = parser.parse_args()
+    args = _parse_args_run_flwr_serverapp().parse_args()
 
     log(INFO, "Starting Flower ServerApp")
-    certificates = try_obtain_certificates(args)
+
+    if not args.insecure:
+        log(
+            ERROR,
+            "`flwr-serverapp` does not support TLS yet. "
+            "Please use the '--insecure' flag.",
+        )
+        sys.exit(1)
 
     log(
         DEBUG,
         "Starting isolated `ServerApp` connected to SuperLink's ServerAppIo API at %s",
-        args.superlink,
+        args.serverappio_api_address,
     )
     run_serverapp(
-        superlink=args.superlink,
+        serverappio_api_address=args.serverappio_api_address,
         log_queue=log_queue,
         run_once=args.run_once,
         flwr_dir=args.flwr_dir,
-        certificates=certificates,
+        certificates=None,
     )
 
     # Restore stdout/stderr
@@ -100,7 +97,7 @@ def flwr_serverapp() -> None:
 
 
 def run_serverapp(  # pylint: disable=R0914, disable=W0212
-    superlink: str,
+    serverappio_api_address: str,
     log_queue: Queue[Optional[str]],
     run_once: bool,
     flwr_dir: Optional[str] = None,
@@ -108,7 +105,7 @@ def run_serverapp(  # pylint: disable=R0914, disable=W0212
 ) -> None:
     """Run Flower ServerApp process."""
     driver = GrpcDriver(
-        serverappio_service_address=superlink,
+        serverappio_service_address=serverappio_api_address,
         root_certificates=certificates,
     )
 
@@ -196,6 +193,12 @@ def run_serverapp(  # pylint: disable=R0914, disable=W0212
             run_status = RunStatus(Status.FINISHED, SubStatus.FAILED, str(ex))
 
         finally:
+            # Stop log uploader for this run and upload final logs
+            if log_uploader:
+                stop_log_uploader(log_queue, log_uploader)
+                log_uploader = None
+
+            # Update run status
             if run_status:
                 run_status_proto = run_status_to_proto(run_status)
                 driver._stub.UpdateRunStatus(
@@ -204,11 +207,28 @@ def run_serverapp(  # pylint: disable=R0914, disable=W0212
                     )
                 )
 
-            # Stop log uploader for this run
-            if log_uploader:
-                stop_log_uploader(log_queue, log_uploader)
-                log_uploader = None
-
         # Stop the loop if `flwr-serverapp` is expected to process a single run
         if run_once:
             break
+
+
+def _parse_args_run_flwr_serverapp() -> argparse.ArgumentParser:
+    """Parse flwr-serverapp command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Run a Flower ServerApp",
+    )
+    parser.add_argument(
+        "--serverappio-api-address",
+        default=SERVERAPPIO_API_DEFAULT_CLIENT_ADDRESS,
+        type=str,
+        help="Address of SuperLink's ServerAppIo API (IPv4, IPv6, or a domain name)."
+        f"By default, it is set to {SERVERAPPIO_API_DEFAULT_CLIENT_ADDRESS}.",
+    )
+    parser.add_argument(
+        "--run-once",
+        action="store_true",
+        help="When set, this process will start a single ServerApp for a pending Run. "
+        "If there is no pending Run, the process will exit.",
+    )
+    add_args_flwr_app_common(parser=parser)
+    return parser

flwr/server/superlink/fleet/rest_rere/rest_api.py

@@ -19,7 +19,7 @@ from __future__ import annotations
 
 import sys
 from collections.abc import Awaitable
-from typing import Callable, TypeVar
+from typing import Callable, TypeVar, cast
 
 from google.protobuf.message import Message as GrpcMessage
 
@@ -39,8 +39,9 @@ from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
 )
 from flwr.proto.run_pb2 import GetRunRequest, GetRunResponse  # pylint: disable=E0611
 from flwr.server.superlink.ffs.ffs import Ffs
+from flwr.server.superlink.ffs.ffs_factory import FfsFactory
 from flwr.server.superlink.fleet.message_handler import message_handler
-from flwr.server.superlink.linkstate import LinkState
+from flwr.server.superlink.linkstate import LinkState, LinkStateFactory
 
 try:
     from starlette.applications import Starlette
@@ -90,7 +91,7 @@ def rest_request_response(
 async def create_node(request: CreateNodeRequest) -> CreateNodeResponse:
     """Create Node."""
     # Get state from app
-    state: LinkState = app.state.STATE_FACTORY.state()
+    state: LinkState = cast(LinkStateFactory, app.state.STATE_FACTORY).state()
 
     # Handle message
     return message_handler.create_node(request=request, state=state)
@@ -100,7 +101,7 @@ async def create_node(request: CreateNodeRequest) -> CreateNodeResponse:
 async def delete_node(request: DeleteNodeRequest) -> DeleteNodeResponse:
     """Delete Node Id."""
     # Get state from app
-    state: LinkState = app.state.STATE_FACTORY.state()
+    state: LinkState = cast(LinkStateFactory, app.state.STATE_FACTORY).state()
 
     # Handle message
     return message_handler.delete_node(request=request, state=state)
@@ -110,7 +111,7 @@ async def delete_node(request: DeleteNodeRequest) -> DeleteNodeResponse:
 async def pull_task_ins(request: PullTaskInsRequest) -> PullTaskInsResponse:
     """Pull TaskIns."""
     # Get state from app
-    state: LinkState = app.state.STATE_FACTORY.state()
+    state: LinkState = cast(LinkStateFactory, app.state.STATE_FACTORY).state()
 
     # Handle message
     return message_handler.pull_task_ins(request=request, state=state)
@@ -121,7 +122,7 @@ async def pull_task_ins(request: PullTaskInsRequest) -> PullTaskInsResponse:
 async def push_task_res(request: PushTaskResRequest) -> PushTaskResResponse:
     """Push TaskRes."""
     # Get state from app
-    state: LinkState = app.state.STATE_FACTORY.state()
+    state: LinkState = cast(LinkStateFactory, app.state.STATE_FACTORY).state()
 
     # Handle message
     return message_handler.push_task_res(request=request, state=state)
@@ -131,7 +132,7 @@ async def push_task_res(request: PushTaskResRequest) -> PushTaskResResponse:
 async def ping(request: PingRequest) -> PingResponse:
     """Ping."""
     # Get state from app
-    state: LinkState = app.state.STATE_FACTORY.state()
+    state: LinkState = cast(LinkStateFactory, app.state.STATE_FACTORY).state()
 
     # Handle message
     return message_handler.ping(request=request, state=state)
@@ -141,7 +142,7 @@ async def ping(request: PingRequest) -> PingResponse:
 async def get_run(request: GetRunRequest) -> GetRunResponse:
     """GetRun."""
     # Get state from app
-    state: LinkState = app.state.STATE_FACTORY.state()
+    state: LinkState = cast(LinkStateFactory, app.state.STATE_FACTORY).state()
 
     # Handle message
     return message_handler.get_run(request=request, state=state)
@@ -151,7 +152,7 @@ async def get_run(request: GetRunRequest) -> GetRunResponse:
 async def get_fab(request: GetFabRequest) -> GetFabResponse:
     """GetRun."""
     # Get ffs from app
-    ffs: Ffs = app.state.FFS_FACTORY.state()
+    ffs: Ffs = cast(FfsFactory, app.state.FFS_FACTORY).ffs()
 
     # Handle message
     return message_handler.get_fab(request=request, ffs=ffs)