flwr-nightly 1.10.0.dev20240707__py3-none-any.whl → 1.10.0.dev20240722__py3-none-any.whl

flwr/server/driver/grpc_driver.py

@@ -16,19 +16,21 @@
 
 import time
 import warnings
-from logging import DEBUG, ERROR, WARNING
-from typing import Iterable, List, Optional, Tuple, cast
+from logging import DEBUG, WARNING
+from typing import Iterable, List, Optional, cast
 
 import grpc
 
 from flwr.common import DEFAULT_TTL, EventType, Message, Metadata, RecordSet, event
 from flwr.common.grpc import create_channel
 from flwr.common.logger import log
-from flwr.common.serde import message_from_taskres, message_to_taskins
+from flwr.common.serde import (
+    message_from_taskres,
+    message_to_taskins,
+    user_config_from_proto,
+)
 from flwr.common.typing import Run
 from flwr.proto.driver_pb2 import (  # pylint: disable=E0611
-    CreateRunRequest,
-    CreateRunResponse,
     GetNodesRequest,
     GetNodesResponse,
     PullTaskResRequest,
@@ -53,167 +55,103 @@ Call `connect()` on the `GrpcDriverStub` instance before calling any of the othe
 """
 
 
-class GrpcDriverStub:
-    """`GrpcDriverStub` provides access to the gRPC Driver API/service.
+class GrpcDriver(Driver):
+    """`GrpcDriver` provides an interface to the Driver API.
 
     Parameters
     ----------
-    driver_service_address : Optional[str]
-        The IPv4 or IPv6 address of the Driver API server.
-        Defaults to `"[::]:9091"`.
+    run_id : int
+        The identifier of the run.
+    driver_service_address : str (default: "[::]:9091")
+        The address (URL, IPv6, IPv4) of the SuperLink Driver API service.
     root_certificates : Optional[bytes] (default: None)
         The PEM-encoded root certificates as a byte string.
         If provided, a secure connection using the certificates will be
         established to an SSL-enabled Flower server.
     """
 
-    def __init__(
+    def __init__(  # pylint: disable=too-many-arguments
         self,
+        run_id: int,
         driver_service_address: str = DEFAULT_SERVER_ADDRESS_DRIVER,
         root_certificates: Optional[bytes] = None,
     ) -> None:
-        self.driver_service_address = driver_service_address
-        self.root_certificates = root_certificates
-        self.channel: Optional[grpc.Channel] = None
-        self.stub: Optional[DriverStub] = None
+        self._run_id = run_id
+        self._addr = driver_service_address
+        self._cert = root_certificates
+        self._run: Optional[Run] = None
+        self._grpc_stub: Optional[DriverStub] = None
+        self._channel: Optional[grpc.Channel] = None
+        self.node = Node(node_id=0, anonymous=True)
 
-    def is_connected(self) -> bool:
-        """Return True if connected to the Driver API server, otherwise False."""
-        return self.channel is not None
+    @property
+    def _is_connected(self) -> bool:
+        """Check if connected to the Driver API server."""
+        return self._channel is not None
+
+    def _connect(self) -> None:
+        """Connect to the Driver API.
 
-    def connect(self) -> None:
-        """Connect to the Driver API."""
+        This will not call GetRun.
+        """
         event(EventType.DRIVER_CONNECT)
-        if self.channel is not None or self.stub is not None:
+        if self._is_connected:
             log(WARNING, "Already connected")
             return
-        self.channel = create_channel(
-            server_address=self.driver_service_address,
-            insecure=(self.root_certificates is None),
-            root_certificates=self.root_certificates,
+        self._channel = create_channel(
+            server_address=self._addr,
+            insecure=(self._cert is None),
+            root_certificates=self._cert,
         )
-        self.stub = DriverStub(self.channel)
-        log(DEBUG, "[Driver] Connected to %s", self.driver_service_address)
+        self._grpc_stub = DriverStub(self._channel)
+        log(DEBUG, "[Driver] Connected to %s", self._addr)
 
-    def disconnect(self) -> None:
+    def _disconnect(self) -> None:
         """Disconnect from the Driver API."""
         event(EventType.DRIVER_DISCONNECT)
-        if self.channel is None or self.stub is None:
+        if not self._is_connected:
             log(DEBUG, "Already disconnected")
             return
-        channel = self.channel
-        self.channel = None
-        self.stub = None
+        channel: grpc.Channel = self._channel
+        self._channel = None
+        self._grpc_stub = None
         channel.close()
         log(DEBUG, "[Driver] Disconnected")
 
-    def create_run(self, req: CreateRunRequest) -> CreateRunResponse:
-        """Request for run ID."""
-        # Check if channel is open
-        if self.stub is None:
-            log(ERROR, ERROR_MESSAGE_DRIVER_NOT_CONNECTED)
-            raise ConnectionError("`GrpcDriverStub` instance not connected")
-
-        # Call Driver API
-        res: CreateRunResponse = self.stub.CreateRun(request=req)
-        return res
-
-    def get_run(self, req: GetRunRequest) -> GetRunResponse:
-        """Get run information."""
-        # Check if channel is open
-        if self.stub is None:
-            log(ERROR, ERROR_MESSAGE_DRIVER_NOT_CONNECTED)
-            raise ConnectionError("`GrpcDriverStub` instance not connected")
-
-        # Call gRPC Driver API
-        res: GetRunResponse = self.stub.GetRun(request=req)
-        return res
-
-    def get_nodes(self, req: GetNodesRequest) -> GetNodesResponse:
-        """Get client IDs."""
-        # Check if channel is open
-        if self.stub is None:
-            log(ERROR, ERROR_MESSAGE_DRIVER_NOT_CONNECTED)
-            raise ConnectionError("`GrpcDriverStub` instance not connected")
-
-        # Call gRPC Driver API
-        res: GetNodesResponse = self.stub.GetNodes(request=req)
-        return res
-
-    def push_task_ins(self, req: PushTaskInsRequest) -> PushTaskInsResponse:
-        """Schedule tasks."""
-        # Check if channel is open
-        if self.stub is None:
-            log(ERROR, ERROR_MESSAGE_DRIVER_NOT_CONNECTED)
-            raise ConnectionError("`GrpcDriverStub` instance not connected")
-
-        # Call gRPC Driver API
-        res: PushTaskInsResponse = self.stub.PushTaskIns(request=req)
-        return res
-
-    def pull_task_res(self, req: PullTaskResRequest) -> PullTaskResResponse:
-        """Get task results."""
-        # Check if channel is open
-        if self.stub is None:
-            log(ERROR, ERROR_MESSAGE_DRIVER_NOT_CONNECTED)
-            raise ConnectionError("`GrpcDriverStub` instance not connected")
-
-        # Call Driver API
-        res: PullTaskResResponse = self.stub.PullTaskRes(request=req)
-        return res
-
-
-class GrpcDriver(Driver):
-    """`Driver` class provides an interface to the Driver API.
-
-    Parameters
-    ----------
-    run_id : int
-        The identifier of the run.
-    stub : Optional[GrpcDriverStub] (default: None)
-        The ``GrpcDriverStub`` instance used to communicate with the SuperLink.
-        If None, an instance connected to "[::]:9091" will be created.
-    """
-
-    def __init__(  # pylint: disable=too-many-arguments
-        self,
-        run_id: int,
-        stub: Optional[GrpcDriverStub] = None,
-    ) -> None:
-        self._run_id = run_id
-        self._run: Optional[Run] = None
-        self.stub = stub if stub is not None else GrpcDriverStub()
-        self.node = Node(node_id=0, anonymous=True)
+    def _init_run(self) -> None:
+        # Check if is initialized
+        if self._run is not None:
+            return
+        # Get the run info
+        req = GetRunRequest(run_id=self._run_id)
+        res: GetRunResponse = self._stub.GetRun(req)
+        if not res.HasField("run"):
+            raise RuntimeError(f"Cannot find the run with ID: {self._run_id}")
+        self._run = Run(
+            run_id=res.run.run_id,
+            fab_id=res.run.fab_id,
+            fab_version=res.run.fab_version,
+            override_config=user_config_from_proto(res.run.override_config),
+        )
 
     @property
     def run(self) -> Run:
         """Run information."""
-        self._get_stub_and_run_id()
-        return Run(**vars(cast(Run, self._run)))
+        self._init_run()
+        return Run(**vars(self._run))
 
-    def _get_stub_and_run_id(self) -> Tuple[GrpcDriverStub, int]:
-        # Check if is initialized
-        if self._run is None:
-            # Connect
-            if not self.stub.is_connected():
-                self.stub.connect()
-            # Get the run info
-            req = GetRunRequest(run_id=self._run_id)
-            res = self.stub.get_run(req)
-            if not res.HasField("run"):
-                raise RuntimeError(f"Cannot find the run with ID: {self._run_id}")
-            self._run = Run(
-                run_id=res.run.run_id,
-                fab_id=res.run.fab_id,
-                fab_version=res.run.fab_version,
-            )
-
-        return self.stub, self._run.run_id
+    @property
+    def _stub(self) -> DriverStub:
+        """Driver stub."""
+        if not self._is_connected:
+            self._connect()
+        return cast(DriverStub, self._grpc_stub)
 
     def _check_message(self, message: Message) -> None:
         # Check if the message is valid
         if not (
-            message.metadata.run_id == cast(Run, self._run).run_id
+            # Assume self._run being initialized
+            message.metadata.run_id == self._run_id
             and message.metadata.src_node_id == self.node.node_id
             and message.metadata.message_id == ""
             and message.metadata.reply_to_message == ""
@@ -234,7 +172,7 @@ class GrpcDriver(Driver):
         This method constructs a new `Message` with given content and metadata.
         The `run_id` and `src_node_id` will be set automatically.
         """
-        _, run_id = self._get_stub_and_run_id()
+        self._init_run()
         if ttl:
             warnings.warn(
                 "A custom TTL was set, but note that the SuperLink does not enforce "
@@ -245,7 +183,7 @@ class GrpcDriver(Driver):
 
         ttl_ = DEFAULT_TTL if ttl is None else ttl
         metadata = Metadata(
-            run_id=run_id,
+            run_id=self._run_id,
             message_id="",  # Will be set by the server
             src_node_id=self.node.node_id,
             dst_node_id=dst_node_id,
@@ -258,9 +196,11 @@ class GrpcDriver(Driver):
 
     def get_node_ids(self) -> List[int]:
         """Get node IDs."""
-        stub, run_id = self._get_stub_and_run_id()
+        self._init_run()
         # Call GrpcDriverStub method
-        res = stub.get_nodes(GetNodesRequest(run_id=run_id))
+        res: GetNodesResponse = self._stub.GetNodes(
+            GetNodesRequest(run_id=self._run_id)
+        )
         return [node.node_id for node in res.nodes]
 
     def push_messages(self, messages: Iterable[Message]) -> Iterable[str]:
@@ -269,7 +209,7 @@ class GrpcDriver(Driver):
         This method takes an iterable of messages and sends each message
         to the node specified in `dst_node_id`.
         """
-        stub, _ = self._get_stub_and_run_id()
+        self._init_run()
         # Construct TaskIns
         task_ins_list: List[TaskIns] = []
         for msg in messages:
@@ -280,7 +220,9 @@ class GrpcDriver(Driver):
             # Add to list
             task_ins_list.append(taskins)
         # Call GrpcDriverStub method
-        res = stub.push_task_ins(PushTaskInsRequest(task_ins_list=task_ins_list))
+        res: PushTaskInsResponse = self._stub.PushTaskIns(
+            PushTaskInsRequest(task_ins_list=task_ins_list)
+        )
         return list(res.task_ids)
 
     def pull_messages(self, message_ids: Iterable[str]) -> Iterable[Message]:
@@ -289,9 +231,9 @@ class GrpcDriver(Driver):
         This method is used to collect messages from the SuperLink that correspond to a
         set of given message IDs.
         """
-        stub, _ = self._get_stub_and_run_id()
+        self._init_run()
         # Pull TaskRes
-        res = stub.pull_task_res(
+        res: PullTaskResResponse = self._stub.PullTaskRes(
             PullTaskResRequest(node=self.node, task_ids=message_ids)
         )
         # Convert TaskRes to Message
@@ -331,7 +273,7 @@ class GrpcDriver(Driver):
     def close(self) -> None:
         """Disconnect from the SuperLink if connected."""
         # Check if `connect` was called before
-        if not self.stub.is_connected():
+        if not self._is_connected:
             return
         # Disconnect
-        self.stub.disconnect()
+        self._disconnect()

flwr/server/run_serverapp.py

@@ -22,13 +22,22 @@ from pathlib import Path
 from typing import Optional
 
 from flwr.common import Context, EventType, RecordSet, event
-from flwr.common.config import get_flwr_dir, get_project_config, get_project_dir
+from flwr.common.config import (
+    get_flwr_dir,
+    get_fused_config,
+    get_project_config,
+    get_project_dir,
+)
 from flwr.common.logger import log, update_console_handler, warn_deprecated_feature
 from flwr.common.object_ref import load_app
-from flwr.proto.driver_pb2 import CreateRunRequest  # pylint: disable=E0611
+from flwr.common.typing import UserConfig
+from flwr.proto.driver_pb2 import (  # pylint: disable=E0611
+    CreateRunRequest,
+    CreateRunResponse,
+)
 
 from .driver import Driver
-from .driver.grpc_driver import GrpcDriver, GrpcDriverStub
+from .driver.grpc_driver import GrpcDriver
 from .server_app import LoadServerAppError, ServerApp
 
 ADDRESS_DRIVER_API = "0.0.0.0:9091"
@@ -37,6 +46,7 @@ ADDRESS_DRIVER_API = "0.0.0.0:9091"
 def run(
     driver: Driver,
     server_app_dir: str,
+    server_app_run_config: UserConfig,
     server_app_attr: Optional[str] = None,
     loaded_server_app: Optional[ServerApp] = None,
 ) -> None:
@@ -69,7 +79,9 @@ def run(
     server_app = _load()
 
     # Initialize Context
-    context = Context(state=RecordSet())
+    context = Context(
+        node_id=0, node_config={}, state=RecordSet(), run_config=server_app_run_config
+    )
 
     # Call ServerApp
     server_app(driver=driver, context=context)
@@ -144,22 +156,29 @@ def run_server_app() -> None:  # pylint: disable=too-many-branches
             "For more details, use: ``flower-server-app -h``"
         )
 
-    stub = GrpcDriverStub(
-        driver_service_address=args.superlink, root_certificates=root_certificates
-    )
+    # Initialize GrpcDriver
     if args.run_id is not None:
         # User provided `--run-id`, but not `server-app`
-        run_id = args.run_id
+        driver = GrpcDriver(
+            run_id=args.run_id,
+            driver_service_address=args.superlink,
+            root_certificates=root_certificates,
+        )
     else:
         # User provided `server-app`, but not `--run-id`
         # Create run if run_id is not provided
-        stub.connect()
+        driver = GrpcDriver(
+            run_id=0,  # Will be overwritten
+            driver_service_address=args.superlink,
+            root_certificates=root_certificates,
+        )
+        # Create run
         req = CreateRunRequest(fab_id=args.fab_id, fab_version=args.fab_version)
-        res = stub.create_run(req)
-        run_id = res.run_id
+        res: CreateRunResponse = driver._stub.CreateRun(req)  # pylint: disable=W0212
+        # Overwrite driver._run_id
+        driver._run_id = res.run_id  # pylint: disable=W0212
 
-    # Initialize GrpcDriver
-    driver = GrpcDriver(run_id=run_id, stub=stub)
+    server_app_run_config = {}
 
     # Dynamically obtain ServerApp path based on run_id
     if args.run_id is not None:
@@ -168,7 +187,8 @@ def run_server_app() -> None:  # pylint: disable=too-many-branches
         run_ = driver.run
         server_app_dir = str(get_project_dir(run_.fab_id, run_.fab_version, flwr_dir))
         config = get_project_config(server_app_dir)
-        server_app_attr = config["flower"]["components"]["serverapp"]
+        server_app_attr = config["tool"]["flwr"]["app"]["components"]["serverapp"]
+        server_app_run_config = get_fused_config(run_, flwr_dir)
     else:
         # User provided `server-app`, but not `--run-id`
         server_app_dir = str(Path(args.dir).absolute())
@@ -182,7 +202,12 @@ def run_server_app() -> None:  # pylint: disable=too-many-branches
     )
 
     # Run the ServerApp with the Driver
-    run(driver=driver, server_app_dir=server_app_dir, server_app_attr=server_app_attr)
+    run(
+        driver=driver,
+        server_app_dir=server_app_dir,
+        server_app_run_config=server_app_run_config,
+        server_app_attr=server_app_attr,
+    )
 
     # Clean up
     driver.close()

flwr/server/server_app.py

@@ -17,8 +17,11 @@
 
 from typing import Callable, Optional
 
-from flwr.common import Context, RecordSet
-from flwr.common.logger import warn_preview_feature
+from flwr.common import Context
+from flwr.common.logger import (
+    warn_deprecated_feature_with_example,
+    warn_preview_feature,
+)
 from flwr.server.strategy import Strategy
 
 from .client_manager import ClientManager
@@ -26,7 +29,20 @@ from .compat import start_driver
 from .driver import Driver
 from .server import Server
 from .server_config import ServerConfig
-from .typing import ServerAppCallable
+from .typing import ServerAppCallable, ServerFn
+
+SERVER_FN_USAGE_EXAMPLE = """
+
+        def server_fn(context: Context):
+            server_config = ServerConfig(num_rounds=3)
+            strategy = FedAvg()
+            return ServerAppComponents(
+                strategy=strategy,
+                server_config=server_config,
+        )
+
+        app = ServerApp(server_fn=server_fn)
+"""
 
 
 class ServerApp:
@@ -36,13 +52,15 @@ class ServerApp:
     --------
     Use the `ServerApp` with an existing `Strategy`:
 
-    >>> server_config = ServerConfig(num_rounds=3)
-    >>> strategy = FedAvg()
+    >>> def server_fn(context: Context):
+    >>>     server_config = ServerConfig(num_rounds=3)
+    >>>     strategy = FedAvg()
+    >>>     return ServerAppComponents(
+    >>>         strategy=strategy,
+    >>>         server_config=server_config,
+    >>>     )
     >>>
-    >>> app = ServerApp(
-    >>>     server_config=server_config,
-    >>>     strategy=strategy,
-    >>> )
+    >>> app = ServerApp(server_fn=server_fn)
 
     Use the `ServerApp` with a custom main function:
 
@@ -53,23 +71,52 @@ class ServerApp:
     >>>    print("ServerApp running")
     """
 
+    # pylint: disable=too-many-arguments
     def __init__(
         self,
         server: Optional[Server] = None,
         config: Optional[ServerConfig] = None,
         strategy: Optional[Strategy] = None,
         client_manager: Optional[ClientManager] = None,
+        server_fn: Optional[ServerFn] = None,
     ) -> None:
+        if any([server, config, strategy, client_manager]):
+            warn_deprecated_feature_with_example(
+                deprecation_message="Passing either `server`, `config`, `strategy` or "
+                "`client_manager` directly to the ServerApp "
+                "constructor is deprecated.",
+                example_message="Pass `ServerApp` arguments wrapped "
+                "in a `flwr.server.ServerAppComponents` object that gets "
+                "returned by a function passed as the `server_fn` argument "
+                "to the `ServerApp` constructor. For example: ",
+                code_example=SERVER_FN_USAGE_EXAMPLE,
+            )
+
+            if server_fn:
+                raise ValueError(
+                    "Passing `server_fn` is incompatible with passing the "
+                    "other arguments (now deprecated) to ServerApp. "
+                    "Use `server_fn` exclusively."
+                )
+
         self._server = server
         self._config = config
         self._strategy = strategy
         self._client_manager = client_manager
+        self._server_fn = server_fn
         self._main: Optional[ServerAppCallable] = None
 
     def __call__(self, driver: Driver, context: Context) -> None:
         """Execute `ServerApp`."""
         # Compatibility mode
         if not self._main:
+            if self._server_fn:
+                # Execute server_fn()
+                components = self._server_fn(context)
+                self._server = components.server
+                self._config = components.config
+                self._strategy = components.strategy
+                self._client_manager = components.client_manager
             start_driver(
                 server=self._server,
                 config=self._config,
@@ -80,7 +127,6 @@ class ServerApp:
             return
 
         # New execution mode
-        context = Context(state=RecordSet())
         self._main(driver, context)
 
     def main(self) -> Callable[[ServerAppCallable], ServerAppCallable]:

flwr/server/serverapp_components.py

@@ -0,0 +1,52 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""ServerAppComponents for the ServerApp."""
+
+
+from dataclasses import dataclass
+from typing import Optional
+
+from .client_manager import ClientManager
+from .server import Server
+from .server_config import ServerConfig
+from .strategy import Strategy
+
+
+@dataclass
+class ServerAppComponents:  # pylint: disable=too-many-instance-attributes
+    """Components to construct a ServerApp.
+
+    Parameters
+    ----------
+    server : Optional[Server] (default: None)
+        A server implementation, either `flwr.server.Server` or a subclass
+        thereof. If no instance is provided, one will be created internally.
+    config : Optional[ServerConfig] (default: None)
+        Currently supported values are `num_rounds` (int, default: 1) and
+        `round_timeout` in seconds (float, default: None).
+    strategy : Optional[Strategy] (default: None)
+        An implementation of the abstract base class
+        `flwr.server.strategy.Strategy`. If no strategy is provided, then
+        `flwr.server.strategy.FedAvg` will be used.
+    client_manager : Optional[ClientManager] (default: None)
+        An implementation of the class `flwr.server.ClientManager`. If no
+        implementation is provided, then `flwr.server.SimpleClientManager`
+        will be used.
+    """
+
+    server: Optional[Server] = None
+    config: Optional[ServerConfig] = None
+    strategy: Optional[Strategy] = None
+    client_manager: Optional[ClientManager] = None

flwr/server/superlink/driver/driver_servicer.py

@@ -23,6 +23,7 @@ from uuid import UUID
 import grpc
 
 from flwr.common.logger import log
+from flwr.common.serde import user_config_from_proto, user_config_to_proto
 from flwr.proto import driver_pb2_grpc  # pylint: disable=E0611
 from flwr.proto.driver_pb2 import (  # pylint: disable=E0611
     CreateRunRequest,
@@ -69,7 +70,11 @@ class DriverServicer(driver_pb2_grpc.DriverServicer):
         """Create run ID."""
         log(DEBUG, "DriverServicer.CreateRun")
         state: State = self.state_factory.state()
-        run_id = state.create_run(request.fab_id, request.fab_version)
+        run_id = state.create_run(
+            request.fab_id,
+            request.fab_version,
+            user_config_from_proto(request.override_config),
+        )
         return CreateRunResponse(run_id=run_id)
 
     def PushTaskIns(
@@ -145,8 +150,18 @@ class DriverServicer(driver_pb2_grpc.DriverServicer):
 
         # Retrieve run information
         run = state.get_run(request.run_id)
-        run_proto = None if run is None else Run(**vars(run))
-        return GetRunResponse(run=run_proto)
+
+        if run is None:
+            return GetRunResponse()
+
+        return GetRunResponse(
+            run=Run(
+                run_id=run.run_id,
+                fab_id=run.fab_id,
+                fab_version=run.fab_version,
+                override_config=user_config_to_proto(run.override_config),
+            )
+        )
 
 
 def _raise_if(validation_error: bool, detail: str) -> None:

flwr/server/superlink/fleet/message_handler/message_handler.py

@@ -19,6 +19,7 @@ import time
 from typing import List, Optional
 from uuid import UUID
 
+from flwr.common.serde import user_config_to_proto
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     CreateNodeResponse,
@@ -113,5 +114,15 @@ def get_run(
 ) -> GetRunResponse:
     """Get run information."""
     run = state.get_run(request.run_id)
-    run_proto = None if run is None else Run(**vars(run))
-    return GetRunResponse(run=run_proto)
+
+    if run is None:
+        return GetRunResponse()
+
+    return GetRunResponse(
+        run=Run(
+            run_id=run.run_id,
+            fab_id=run.fab_id,
+            fab_version=run.fab_version,
+            override_config=user_config_to_proto(run.override_config),
+        )
+    )