flwr-nightly 1.10.0.dev20240707__py3-none-any.whl → 1.11.0.dev20240724__py3-none-any.whl

flwr/superexec/deployment.py

@@ -15,8 +15,8 @@
 """Deployment engine executor."""
 
 import subprocess
-import sys
 from logging import ERROR, INFO
+from pathlib import Path
 from typing import Optional
 
 from typing_extensions import override
@@ -25,6 +25,8 @@ from flwr.cli.config_utils import get_fab_metadata
 from flwr.cli.install import install_from_fab
 from flwr.common.grpc import create_channel
 from flwr.common.logger import log
+from flwr.common.serde import user_config_to_proto
+from flwr.common.typing import UserConfig
 from flwr.proto.driver_pb2 import CreateRunRequest  # pylint: disable=E0611
 from flwr.proto.driver_pb2_grpc import DriverStub
 from flwr.server.driver.grpc_driver import DEFAULT_SERVER_ADDRESS_DRIVER
@@ -33,65 +35,139 @@ from .executor import Executor, RunTracker
 
 
 class DeploymentEngine(Executor):
-    """Deployment engine executor."""
+    """Deployment engine executor.
+
+    Parameters
+    ----------
+    superlink: str (default: "0.0.0.0:9091")
+        Address of the SuperLink to connect to.
+    root_certificates: Optional[str] (default: None)
+        Specifies the path to the PEM-encoded root certificate file for
+        establishing secure HTTPS connections.
+    flwr_dir: Optional[str] (default: None)
+        The path containing installed Flower Apps.
+    """
 
     def __init__(
         self,
-        address: str = DEFAULT_SERVER_ADDRESS_DRIVER,
-        root_certificates: Optional[bytes] = None,
+        superlink: str = DEFAULT_SERVER_ADDRESS_DRIVER,
+        root_certificates: Optional[str] = None,
+        flwr_dir: Optional[str] = None,
     ) -> None:
-        self.address = address
-        self.root_certificates = root_certificates
+        self.superlink = superlink
+        if root_certificates is None:
+            self.root_certificates = None
+            self.root_certificates_bytes = None
+        else:
+            self.root_certificates = root_certificates
+            self.root_certificates_bytes = Path(root_certificates).read_bytes()
+        self.flwr_dir = flwr_dir
         self.stub: Optional[DriverStub] = None
 
-    def _connect(self) -> None:
-        if self.stub is None:
-            channel = create_channel(
-                server_address=self.address,
-                insecure=(self.root_certificates is None),
-                root_certificates=self.root_certificates,
-            )
-            self.stub = DriverStub(channel)
+    @override
+    def set_config(
+        self,
+        config: UserConfig,
+    ) -> None:
+        """Set executor config arguments.
+
+        Parameters
+        ----------
+        config : UserConfig
+            A dictionary for configuration values.
+            Supported configuration key/value pairs:
+            - "superlink": str
+                The address of the SuperLink Driver API.
+            - "root-certificates": str
+                The path to the root certificates.
+            - "flwr-dir": str
+                The path to the Flower directory.
+        """
+        if not config:
+            return
+        if superlink_address := config.get("superlink"):
+            if not isinstance(superlink_address, str):
+                raise ValueError("The `superlink` value should be of type `str`.")
+            self.superlink = superlink_address
+        if root_certificates := config.get("root-certificates"):
+            if not isinstance(root_certificates, str):
+                raise ValueError(
+                    "The `root-certificates` value should be of type `str`."
+                )
+            self.root_certificates = root_certificates
+            self.root_certificates_bytes = Path(str(root_certificates)).read_bytes()
+        if flwr_dir := config.get("flwr-dir"):
+            if not isinstance(flwr_dir, str):
+                raise ValueError("The `flwr-dir` value should be of type `str`.")
+            self.flwr_dir = str(flwr_dir)
 
-    def _create_run(self, fab_id: str, fab_version: str) -> int:
+    def _connect(self) -> None:
+        if self.stub is not None:
+            return
+        channel = create_channel(
+            server_address=self.superlink,
+            insecure=(self.root_certificates_bytes is None),
+            root_certificates=self.root_certificates_bytes,
+        )
+        self.stub = DriverStub(channel)
+
+    def _create_run(
+        self,
+        fab_id: str,
+        fab_version: str,
+        override_config: UserConfig,
+    ) -> int:
         if self.stub is None:
             self._connect()
 
         assert self.stub is not None
 
-        req = CreateRunRequest(fab_id=fab_id, fab_version=fab_version)
+        req = CreateRunRequest(
+            fab_id=fab_id,
+            fab_version=fab_version,
+            override_config=user_config_to_proto(override_config),
+        )
         res = self.stub.CreateRun(request=req)
         return int(res.run_id)
 
     @override
-    def start_run(self, fab_file: bytes) -> Optional[RunTracker]:
+    def start_run(
+        self,
+        fab_file: bytes,
+        override_config: UserConfig,
+        federation_config: UserConfig,
+    ) -> Optional[RunTracker]:
         """Start run using the Flower Deployment Engine."""
         try:
             # Install FAB to flwr dir
             fab_version, fab_id = get_fab_metadata(fab_file)
-            fab_path = install_from_fab(fab_file, None, True)
-
-            # Install FAB Python package
-            subprocess.check_call(
-                [sys.executable, "-m", "pip", "install", str(fab_path)],
-                stdout=subprocess.DEVNULL,
-                stderr=subprocess.DEVNULL,
-            )
+            install_from_fab(fab_file, None, True)
 
             # Call SuperLink to create run
-            run_id: int = self._create_run(fab_id, fab_version)
+            run_id: int = self._create_run(fab_id, fab_version, override_config)
             log(INFO, "Created run %s", str(run_id))
 
-            # Start ServerApp
+            command = [
+                "flower-server-app",
+                "--run-id",
+                str(run_id),
+                "--superlink",
+                str(self.superlink),
+            ]
+
+            if self.flwr_dir:
+                command.append("--flwr-dir")
+                command.append(self.flwr_dir)
+
+            if self.root_certificates is None:
+                command.append("--insecure")
+            else:
+                command.append("--root-certificates")
+                command.append(self.root_certificates)
+
+            # Execute the command
             proc = subprocess.Popen(  # pylint: disable=consider-using-with
-                [
-                    "flower-server-app",
-                    "--run-id",
-                    str(run_id),
-                    "--insecure",
-                ],
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
+                command,
                 text=True,
             )
             log(INFO, "Started run %s", str(run_id))

flwr/superexec/exec_grpc.py

@@ -21,6 +21,7 @@ import grpc
 
 from flwr.common import GRPC_MAX_MESSAGE_LENGTH
 from flwr.common.logger import log
+from flwr.common.typing import UserConfig
 from flwr.proto.exec_pb2_grpc import add_ExecServicer_to_server
 from flwr.server.superlink.fleet.grpc_bidi.grpc_server import generic_create_grpc_server
 
@@ -32,8 +33,11 @@ def run_superexec_api_grpc(
     address: str,
     executor: Executor,
     certificates: Optional[Tuple[bytes, bytes, bytes]],
+    config: UserConfig,
 ) -> grpc.Server:
     """Run SuperExec API (gRPC, request-response)."""
+    executor.set_config(config)
+
     exec_servicer: grpc.Server = ExecServicer(
         executor=executor,
     )
@@ -45,7 +49,7 @@ def run_superexec_api_grpc(
         certificates=certificates,
     )
 
-    log(INFO, "Flower ECE: Starting SuperExec API (gRPC-rere) on %s", address)
+    log(INFO, "Starting Flower SuperExec gRPC server on %s", address)
     superexec_grpc_server.start()
 
     return superexec_grpc_server

flwr/superexec/exec_servicer.py

@@ -21,6 +21,7 @@ from typing import Any, Dict, Generator
 import grpc
 
 from flwr.common.logger import log
+from flwr.common.serde import user_config_from_proto
 from flwr.proto import exec_pb2_grpc  # pylint: disable=E0611
 from flwr.proto.exec_pb2 import (  # pylint: disable=E0611
     StartRunRequest,
@@ -45,7 +46,11 @@ class ExecServicer(exec_pb2_grpc.ExecServicer):
         """Create run ID."""
         log(INFO, "ExecServicer.StartRun")
 
-        run = self.executor.start_run(request.fab_file)
+        run = self.executor.start_run(
+            request.fab_file,
+            user_config_from_proto(request.override_config),
+            user_config_from_proto(request.federation_config),
+        )
 
         if run is None:
             log(ERROR, "Executor failed to start run")

flwr/superexec/executor.py

@@ -19,6 +19,8 @@ from dataclasses import dataclass
 from subprocess import Popen
 from typing import Optional
 
+from flwr.common.typing import UserConfig
+
 
 @dataclass
 class RunTracker:
@@ -31,10 +33,25 @@ class RunTracker:
 class Executor(ABC):
     """Execute and monitor a Flower run."""
 
+    @abstractmethod
+    def set_config(
+        self,
+        config: UserConfig,
+    ) -> None:
+        """Register provided config as class attributes.
+
+        Parameters
+        ----------
+        config : UserConfig
+            A dictionary for configuration values.
+        """
+
     @abstractmethod
     def start_run(
         self,
         fab_file: bytes,
+        override_config: UserConfig,
+        federation_config: UserConfig,
     ) -> Optional[RunTracker]:
         """Start a run using the given Flower FAB ID and version.
 
@@ -45,6 +62,10 @@ class Executor(ABC):
         ----------
         fab_file : bytes
             The Flower App Bundle file bytes.
+        override_config: UserConfig
+            The config overrides dict sent by the user (using `flwr run`).
+        federation_config: UserConfig
+            The federation options dict sent by the user (using `flwr run`).
 
         Returns
         -------

flwr/superexec/simulation.py

@@ -0,0 +1,181 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Simulation engine executor."""
+
+
+import subprocess
+import sys
+from logging import ERROR, INFO, WARN
+from typing import Optional
+
+from typing_extensions import override
+
+from flwr.cli.config_utils import load_and_validate
+from flwr.cli.install import install_from_fab
+from flwr.common.constant import RUN_ID_NUM_BYTES
+from flwr.common.logger import log
+from flwr.common.typing import UserConfig
+from flwr.server.superlink.state.utils import generate_rand_int_from_bytes
+
+from .executor import Executor, RunTracker
+
+
+def _user_config_to_str(user_config: UserConfig) -> str:
+    """Convert override user config to string."""
+    user_config_list_str = []
+    for key, value in user_config.items():
+        if isinstance(value, bool):
+            user_config_list_str.append(f"{key}={str(value).lower()}")
+        elif isinstance(value, (int, float)):
+            user_config_list_str.append(f"{key}={value}")
+        elif isinstance(value, str):
+            user_config_list_str.append(f'{key}="{value}"')
+        else:
+            raise ValueError(
+                "Only types `bool`, `float`, `int` and `str` are supported"
+            )
+
+    user_config_str = ",".join(user_config_list_str)
+    return user_config_str
+
+
+class SimulationEngine(Executor):
+    """Simulation engine executor.
+
+    Parameters
+    ----------
+    num_supernodes: Opitonal[str] (default: None)
+        Total number of nodes to involve in the simulation.
+    """
+
+    def __init__(
+        self,
+        num_supernodes: Optional[int] = None,
+    ) -> None:
+        self.num_supernodes = num_supernodes
+
+    @override
+    def set_config(
+        self,
+        config: UserConfig,
+    ) -> None:
+        """Set executor config arguments.
+
+        Parameters
+        ----------
+        config : UserConfig
+            A dictionary for configuration values.
+            Supported configuration key/value pairs:
+            - "num-supernodes": int
+                Number of nodes to register for the simulation.
+        """
+        if num_supernodes := config.get("num-supernodes"):
+            if not isinstance(num_supernodes, int):
+                raise ValueError("The `num-supernodes` value should be of type `int`.")
+            self.num_supernodes = num_supernodes
+        elif self.num_supernodes is None:
+            log(
+                ERROR,
+                "To start a run with the simulation plugin, please specify "
+                "the number of SuperNodes. This can be done by using the "
+                "`--executor-config` argument when launching the SuperExec.",
+            )
+            raise ValueError(
+                "`num-supernodes` must not be `None`, it must be a valid "
+                "positive integer."
+            )
+
+    @override
+    def start_run(
+        self,
+        fab_file: bytes,
+        override_config: UserConfig,
+        federation_config: UserConfig,
+    ) -> Optional[RunTracker]:
+        """Start run using the Flower Simulation Engine."""
+        if self.num_supernodes is None:
+            raise ValueError(
+                "Error in `SuperExec` (`SimulationEngine` executor):\n\n"
+                "`num-supernodes` must not be `None`, it must be a valid "
+                "positive integer. In order to start this simulation executor "
+                "with a specified number of `SuperNodes`, you can either provide "
+                "a `--executor` that has been initialized with a number of nodes "
+                "to the `flower-superexec` CLI, or `--executor-config num-supernodes=N`"
+                "to the `flower-superexec` CLI."
+            )
+        try:
+
+            # Install FAB to flwr dir
+            fab_path = install_from_fab(fab_file, None, True)
+
+            # Install FAB Python package
+            subprocess.check_call(
+                [sys.executable, "-m", "pip", "install", "--no-deps", str(fab_path)],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+
+            # Load and validate config
+            config, errors, warnings = load_and_validate(fab_path / "pyproject.toml")
+            if errors:
+                raise ValueError(errors)
+
+            if warnings:
+                log(WARN, warnings)
+
+            if config is None:
+                raise ValueError(
+                    "Config extracted from FAB's pyproject.toml is not valid"
+                )
+
+            # In Simulation there is no SuperLink, still we create a run_id
+            run_id = generate_rand_int_from_bytes(RUN_ID_NUM_BYTES)
+            log(INFO, "Created run %s", str(run_id))
+
+            # Prepare commnand
+            command = [
+                "flower-simulation",
+                "--app",
+                f"{str(fab_path)}",
+                "--num-supernodes",
+                f"{federation_config.get('num-supernodes', self.num_supernodes)}",
+                "--run-id",
+                str(run_id),
+            ]
+
+            if override_config:
+                override_config_str = _user_config_to_str(override_config)
+                command.extend(["--run-config", f"{override_config_str}"])
+
+            # Start Simulation
+            proc = subprocess.Popen(  # pylint: disable=consider-using-with
+                command,
+                text=True,
+            )
+
+            log(INFO, "Started run %s", str(run_id))
+
+            return RunTracker(
+                run_id=run_id,
+                proc=proc,
+            )
+
+        # pylint: disable-next=broad-except
+        except Exception as e:
+            log(ERROR, "Could not start run: %s", str(e))
+            return None
+
+
+executor = SimulationEngine()

{flwr_nightly-1.10.0.dev20240707.dist-info → flwr_nightly-1.11.0.dev20240724.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.10.0.dev20240707
+Version: 1.11.0.dev20240724
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0
@@ -33,7 +33,7 @@ Classifier: Typing :: Typed
 Provides-Extra: rest
 Provides-Extra: simulation
 Requires-Dist: cryptography (>=42.0.4,<43.0.0)
-Requires-Dist: grpcio (>=1.60.0,<2.0.0)
+Requires-Dist: grpcio (>=1.60.0,<2.0.0,!=1.64.2,!=1.65.1)
 Requires-Dist: iterators (>=0.0.2,<0.0.3)
 Requires-Dist: numpy (>=1.21.0,<2.0.0)
 Requires-Dist: pathspec (>=0.12.1,<0.13.0)
@@ -43,6 +43,7 @@ Requires-Dist: ray (==2.10.0) ; (python_version >= "3.8" and python_version < "3
 Requires-Dist: requests (>=2.31.0,<3.0.0) ; extra == "rest"
 Requires-Dist: starlette (>=0.31.0,<0.32.0) ; extra == "rest"
 Requires-Dist: tomli (>=2.0.1,<3.0.0)
+Requires-Dist: tomli-w (>=1.0.0,<2.0.0)
 Requires-Dist: typer[all] (>=0.9.0,<0.10.0)
 Requires-Dist: uvicorn[standard] (>=0.23.0,<0.24.0) ; extra == "rest"
 Project-URL: Documentation, https://flower.ai