flwr-nightly 1.10.0.dev20240709__py3-none-any.whl → 1.10.0.dev20240711__py3-none-any.whl

flwr/cli/config_utils.py

@@ -108,6 +108,14 @@ def load(path: Optional[Path] = None) -> Optional[Dict[str, Any]]:
         return load_from_string(toml_file.read())
 
 
+def _validate_run_config(config_dict: Dict[str, Any], errors: List[str]) -> None:
+    for key, value in config_dict.items():
+        if isinstance(value, dict):
+            _validate_run_config(config_dict[key], errors)
+        elif not isinstance(value, str):
+            errors.append(f"Config value of key {key} is not of type `str`.")
+
+
 # pylint: disable=too-many-branches
 def validate_fields(config: Dict[str, Any]) -> Tuple[bool, List[str], List[str]]:
     """Validate pyproject.toml fields."""
@@ -133,6 +141,8 @@ def validate_fields(config: Dict[str, Any]) -> Tuple[bool, List[str], List[str]]
     else:
         if "publisher" not in config["flower"]:
             errors.append('Property "publisher" missing in [flower]')
+        if "config" in config["flower"]:
+            _validate_run_config(config["flower"]["config"], errors)
         if "components" not in config["flower"]:
             errors.append("Missing [flower.components] section")
         else:

flwr/cli/new/new.py

@@ -264,9 +264,11 @@ def new(
             bold=True,
         )
     )
+
+    _add = "	huggingface-cli login\n" if framework_str == "flowertune" else ""
     print(
         typer.style(
-            f"	cd {package_name}\n" + "	pip install -e .\n	flwr run\n",
+            f"	cd {package_name}\n" + "	pip install -e .\n" + _add + "	flwr run\n",
             fg=typer.colors.BRIGHT_CYAN,
             bold=True,
         )

flwr/cli/run/run.py

@@ -18,13 +18,14 @@ import sys
 from enum import Enum
 from logging import DEBUG
 from pathlib import Path
-from typing import Optional
+from typing import Dict, Optional
 
 import typer
 from typing_extensions import Annotated
 
 from flwr.cli import config_utils
 from flwr.cli.build import build
+from flwr.common.config import parse_config_args
 from flwr.common.constant import SUPEREXEC_DEFAULT_ADDRESS
 from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel
 from flwr.common.logger import log
@@ -58,15 +59,20 @@ def run(
         Optional[Path],
         typer.Option(help="Path of the Flower project to run"),
     ] = None,
+    config_overrides: Annotated[
+        Optional[str],
+        typer.Option(
+            "--config",
+            "-c",
+            help="Override configuration key-value pairs",
+        ),
+    ] = None,
 ) -> None:
     """Run Flower project."""
-    if use_superexec:
-        _start_superexec_run(directory)
-        return
-
     typer.secho("Loading project configuration... ", fg=typer.colors.BLUE)
 
-    config, errors, warnings = config_utils.load_and_validate()
+    pyproject_path = directory / "pyproject.toml" if directory else None
+    config, errors, warnings = config_utils.load_and_validate(path=pyproject_path)
 
     if config is None:
         typer.secho(
@@ -88,6 +94,12 @@ def run(
 
     typer.secho("Success", fg=typer.colors.GREEN)
 
+    if use_superexec:
+        _start_superexec_run(
+            parse_config_args(config_overrides, separator=","), directory
+        )
+        return
+
     server_app_ref = config["flower"]["components"]["serverapp"]
     client_app_ref = config["flower"]["components"]["clientapp"]
 
@@ -115,7 +127,9 @@ def run(
         )
 
 
-def _start_superexec_run(directory: Optional[Path]) -> None:
+def _start_superexec_run(
+    override_config: Dict[str, str], directory: Optional[Path]
+) -> None:
     def on_channel_state_change(channel_connectivity: str) -> None:
         """Log channel connectivity."""
         log(DEBUG, channel_connectivity)
@@ -132,6 +146,9 @@ def _start_superexec_run(directory: Optional[Path]) -> None:
 
     fab_path = build(directory)
 
-    req = StartRunRequest(fab_file=Path(fab_path).read_bytes())
+    req = StartRunRequest(
+        fab_file=Path(fab_path).read_bytes(),
+        override_config=override_config,
+    )
     res = stub.StartRun(req)
     typer.secho(f"🎊 Successfully started run {res.run_id}", fg=typer.colors.GREEN)

flwr/client/app.py

@@ -19,6 +19,7 @@ import sys
 import time
 from dataclasses import dataclass
 from logging import DEBUG, ERROR, INFO, WARN
+from pathlib import Path
 from typing import Callable, ContextManager, Dict, Optional, Tuple, Type, Union
 
 from cryptography.hazmat.primitives.asymmetric import ec
@@ -193,6 +194,7 @@ def _start_client_internal(
     max_retries: Optional[int] = None,
     max_wait_time: Optional[float] = None,
     partition_id: Optional[int] = None,
+    flwr_dir: Optional[Path] = None,
 ) -> None:
     """Start a Flower client node which connects to a Flower server.
 
@@ -239,6 +241,8 @@ def _start_client_internal(
     partition_id: Optional[int] (default: None)
         The data partition index associated with this node. Better suited for
         prototyping purposes.
+    flwr_dir: Optional[Path] (default: None)
+        The fully resolved path containing installed Flower Apps.
     """
     if insecure is None:
         insecure = root_certificates is None
@@ -316,7 +320,7 @@ def _start_client_internal(
     )
 
     node_state = NodeState(partition_id=partition_id)
-    run_info: Dict[int, Run] = {}
+    runs: Dict[int, Run] = {}
 
     while not app_state_tracker.interrupt:
         sleep_duration: int = 0
@@ -366,15 +370,17 @@ def _start_client_internal(
 
                     # Get run info
                     run_id = message.metadata.run_id
-                    if run_id not in run_info:
+                    if run_id not in runs:
                         if get_run is not None:
-                            run_info[run_id] = get_run(run_id)
+                            runs[run_id] = get_run(run_id)
                         # If get_run is None, i.e., in grpc-bidi mode
                         else:
-                            run_info[run_id] = Run(run_id, "", "", {})
+                            runs[run_id] = Run(run_id, "", "", {})
 
                     # Register context for this run
-                    node_state.register_context(run_id=run_id)
+                    node_state.register_context(
+                        run_id=run_id, run=runs[run_id], flwr_dir=flwr_dir
+                    )
 
                     # Retrieve context for this run
                     context = node_state.retrieve_context(run_id=run_id)
@@ -388,7 +394,7 @@ def _start_client_internal(
                     # Handle app loading and task message
                     try:
                         # Load ClientApp instance
-                        run: Run = run_info[run_id]
+                        run: Run = runs[run_id]
                         client_app: ClientApp = load_client_app_fn(
                             run.fab_id, run.fab_version
                         )

flwr/client/node_state.py

@@ -15,9 +15,21 @@
 """Node state."""
 
 
+from dataclasses import dataclass
+from pathlib import Path
 from typing import Any, Dict, Optional
 
 from flwr.common import Context, RecordSet
+from flwr.common.config import get_fused_config
+from flwr.common.typing import Run
+
+
+@dataclass()
+class RunInfo:
+    """Contains the Context and initial run_config of a Run."""
+
+    context: Context
+    initial_run_config: Dict[str, str]
 
 
 class NodeState:
@@ -25,20 +37,31 @@ class NodeState:
 
     def __init__(self, partition_id: Optional[int]) -> None:
         self._meta: Dict[str, Any] = {}  # holds metadata about the node
-        self.run_contexts: Dict[int, Context] = {}
+        self.run_infos: Dict[int, RunInfo] = {}
         self._partition_id = partition_id
 
-    def register_context(self, run_id: int) -> None:
+    def register_context(
+        self,
+        run_id: int,
+        run: Optional[Run] = None,
+        flwr_dir: Optional[Path] = None,
+    ) -> None:
         """Register new run context for this node."""
-        if run_id not in self.run_contexts:
-            self.run_contexts[run_id] = Context(
-                state=RecordSet(), run_config={}, partition_id=self._partition_id
+        if run_id not in self.run_infos:
+            initial_run_config = get_fused_config(run, flwr_dir) if run else {}
+            self.run_infos[run_id] = RunInfo(
+                initial_run_config=initial_run_config,
+                context=Context(
+                    state=RecordSet(),
+                    run_config=initial_run_config.copy(),
+                    partition_id=self._partition_id,
+                ),
             )
 
     def retrieve_context(self, run_id: int) -> Context:
         """Get run context given a run_id."""
-        if run_id in self.run_contexts:
-            return self.run_contexts[run_id]
+        if run_id in self.run_infos:
+            return self.run_infos[run_id].context
 
         raise RuntimeError(
             f"Context for run_id={run_id} doesn't exist."
@@ -48,4 +71,9 @@ class NodeState:
 
     def update_context(self, run_id: int, context: Context) -> None:
         """Update run context."""
-        self.run_contexts[run_id] = context
+        if context.run_config != self.run_infos[run_id].initial_run_config:
+            raise ValueError(
+                "The `run_config` field of the `Context` object cannot be "
+                f"modified (run_id: {run_id})."
+            )
+        self.run_infos[run_id].context = context

flwr/client/node_state_tests.py

@@ -59,7 +59,8 @@ def test_multirun_in_node_state() -> None:
         node_state.update_context(run_id=run_id, context=updated_state)
 
     # Verify values
-    for run_id, context in node_state.run_contexts.items():
+    for run_id, run_info in node_state.run_infos.items():
         assert (
-            context.state.configs_records["counter"]["count"] == expected_values[run_id]
+            run_info.context.state.configs_records["counter"]["count"]
+            == expected_values[run_id]
         )

flwr/client/supernode/app.py

@@ -68,6 +68,7 @@ def run_supernode() -> None:
         max_retries=args.max_retries,
         max_wait_time=args.max_wait_time,
         partition_id=args.partition_id,
+        flwr_dir=get_flwr_dir(args.flwr_dir),
     )
 
     # Graceful shutdown
@@ -178,7 +179,7 @@ def _get_load_client_app_fn(
         else:
             flwr_dir = Path(args.flwr_dir).absolute()
 
-    sys.path.insert(0, str(flwr_dir.absolute()))
+    inserted_path = None
 
     default_app_ref: str = getattr(args, "client-app")
 
@@ -188,6 +189,11 @@ def _get_load_client_app_fn(
             "Flower SuperNode will load and validate ClientApp `%s`",
             getattr(args, "client-app"),
         )
+        # Insert sys.path
+        dir_path = Path(args.dir).absolute()
+        sys.path.insert(0, str(dir_path))
+        inserted_path = str(dir_path)
+
         valid, error_msg = validate(default_app_ref)
         if not valid and error_msg:
             raise LoadClientAppError(error_msg) from None
@@ -196,7 +202,7 @@ def _get_load_client_app_fn(
         # If multi-app feature is disabled
         if not multi_app:
             # Get sys path to be inserted
-            sys_path = Path(args.dir).absolute()
+            dir_path = Path(args.dir).absolute()
 
             # Set app reference
             client_app_ref = default_app_ref
@@ -209,7 +215,7 @@ def _get_load_client_app_fn(
 
             log(WARN, "FAB ID is not provided; the default ClientApp will be loaded.")
             # Get sys path to be inserted
-            sys_path = Path(args.dir).absolute()
+            dir_path = Path(args.dir).absolute()
 
             # Set app reference
             client_app_ref = default_app_ref
@@ -222,13 +228,21 @@ def _get_load_client_app_fn(
                 raise LoadClientAppError("Failed to load ClientApp") from e
 
             # Get sys path to be inserted
-            sys_path = Path(project_dir).absolute()
+            dir_path = Path(project_dir).absolute()
 
             # Set app reference
             client_app_ref = config["flower"]["components"]["clientapp"]
 
         # Set sys.path
-        sys.path.insert(0, str(sys_path))
+        nonlocal inserted_path
+        if inserted_path != str(dir_path):
+            # Remove the previously inserted path
+            if inserted_path is not None:
+                sys.path.remove(inserted_path)
+            # Insert the new path
+            sys.path.insert(0, str(dir_path))
+
+        inserted_path = str(dir_path)
 
         # Load ClientApp
         log(
@@ -236,7 +250,7 @@ def _get_load_client_app_fn(
             "Loading ClientApp `%s`",
             client_app_ref,
         )
-        client_app = load_app(client_app_ref, LoadClientAppError, sys_path)
+        client_app = load_app(client_app_ref, LoadClientAppError, dir_path)
 
         if not isinstance(client_app, ClientApp):
             raise LoadClientAppError(

flwr/common/logger.py

@@ -197,6 +197,31 @@ def warn_deprecated_feature(name: str) -> None:
     )
 
 
+def warn_deprecated_feature_with_example(
+    deprecation_message: str, example_message: str, code_example: str
+) -> None:
+    """Warn if a feature is deprecated and show code example."""
+    log(
+        WARN,
+        """DEPRECATED FEATURE: %s
+
+            Check the following `FEATURE UPDATE` warning message for the preferred
+            new mechanism to use this feature in Flower.
+        """,
+        deprecation_message,
+    )
+    log(
+        WARN,
+        """FEATURE UPDATE: %s
+        ------------------------------------------------------------
+        %s
+        ------------------------------------------------------------
+        """,
+        example_message,
+        code_example,
+    )
+
+
 def warn_unsupported_feature(name: str) -> None:
     """Warn the user when they use an unsupported feature."""
     log(

flwr/server/__init__.py

@@ -28,6 +28,7 @@ from .run_serverapp import run_server_app as run_server_app
 from .server import Server as Server
 from .server_app import ServerApp as ServerApp
 from .server_config import ServerConfig as ServerConfig
+from .serverapp_components import ServerAppComponents as ServerAppComponents
 
 __all__ = [
     "ClientManager",
@@ -36,6 +37,7 @@ __all__ = [
     "LegacyContext",
     "Server",
     "ServerApp",
+    "ServerAppComponents",
     "ServerConfig",
     "SimpleClientManager",
     "run_server_app",

flwr/server/server_app.py

@@ -17,8 +17,11 @@
 
 from typing import Callable, Optional
 
-from flwr.common import Context, RecordSet
-from flwr.common.logger import warn_preview_feature
+from flwr.common import Context
+from flwr.common.logger import (
+    warn_deprecated_feature_with_example,
+    warn_preview_feature,
+)
 from flwr.server.strategy import Strategy
 
 from .client_manager import ClientManager
@@ -26,7 +29,20 @@ from .compat import start_driver
 from .driver import Driver
 from .server import Server
 from .server_config import ServerConfig
-from .typing import ServerAppCallable
+from .typing import ServerAppCallable, ServerFn
+
+SERVER_FN_USAGE_EXAMPLE = """
+
+        def server_fn(context: Context):
+            server_config = ServerConfig(num_rounds=3)
+            strategy = FedAvg()
+            return ServerAppComponents(
+                strategy=strategy,
+                server_config=server_config,
+        )
+
+        app = ServerApp(server_fn=server_fn)
+"""
 
 
 class ServerApp:
@@ -36,13 +52,15 @@ class ServerApp:
     --------
     Use the `ServerApp` with an existing `Strategy`:
 
-    >>> server_config = ServerConfig(num_rounds=3)
-    >>> strategy = FedAvg()
+    >>> def server_fn(context: Context):
+    >>>     server_config = ServerConfig(num_rounds=3)
+    >>>     strategy = FedAvg()
+    >>>     return ServerAppComponents(
+    >>>         strategy=strategy,
+    >>>         server_config=server_config,
+    >>>     )
     >>>
-    >>> app = ServerApp(
-    >>>     server_config=server_config,
-    >>>     strategy=strategy,
-    >>> )
+    >>> app = ServerApp(server_fn=server_fn)
 
     Use the `ServerApp` with a custom main function:
 
@@ -53,23 +71,52 @@ class ServerApp:
     >>>    print("ServerApp running")
     """
 
+    # pylint: disable=too-many-arguments
     def __init__(
         self,
         server: Optional[Server] = None,
         config: Optional[ServerConfig] = None,
         strategy: Optional[Strategy] = None,
         client_manager: Optional[ClientManager] = None,
+        server_fn: Optional[ServerFn] = None,
     ) -> None:
+        if any([server, config, strategy, client_manager]):
+            warn_deprecated_feature_with_example(
+                deprecation_message="Passing either `server`, `config`, `strategy` or "
+                "`client_manager` directly to the ServerApp "
+                "constructor is deprecated.",
+                example_message="Pass `ServerApp` arguments wrapped "
+                "in a `flwr.server.ServerAppComponents` object that gets "
+                "returned by a function passed as the `server_fn` argument "
+                "to the `ServerApp` constructor. For example: ",
+                code_example=SERVER_FN_USAGE_EXAMPLE,
+            )
+
+            if server_fn:
+                raise ValueError(
+                    "Passing `server_fn` is incompatible with passing the "
+                    "other arguments (now deprecated) to ServerApp. "
+                    "Use `server_fn` exclusively."
+                )
+
         self._server = server
         self._config = config
         self._strategy = strategy
         self._client_manager = client_manager
+        self._server_fn = server_fn
         self._main: Optional[ServerAppCallable] = None
 
     def __call__(self, driver: Driver, context: Context) -> None:
         """Execute `ServerApp`."""
         # Compatibility mode
         if not self._main:
+            if self._server_fn:
+                # Execute server_fn()
+                components = self._server_fn(context)
+                self._server = components.server
+                self._config = components.config
+                self._strategy = components.strategy
+                self._client_manager = components.client_manager
             start_driver(
                 server=self._server,
                 config=self._config,
@@ -80,7 +127,6 @@ class ServerApp:
             return
 
         # New execution mode
-        context = Context(state=RecordSet(), run_config={})
         self._main(driver, context)
 
     def main(self) -> Callable[[ServerAppCallable], ServerAppCallable]:

flwr/server/serverapp_components.py

@@ -0,0 +1,52 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""ServerAppComponents for the ServerApp."""
+
+
+from dataclasses import dataclass
+from typing import Optional
+
+from .client_manager import ClientManager
+from .server import Server
+from .server_config import ServerConfig
+from .strategy import Strategy
+
+
+@dataclass
+class ServerAppComponents:  # pylint: disable=too-many-instance-attributes
+    """Components to construct a ServerApp.
+
+    Parameters
+    ----------
+    server : Optional[Server] (default: None)
+        A server implementation, either `flwr.server.Server` or a subclass
+        thereof. If no instance is provided, one will be created internally.
+    config : Optional[ServerConfig] (default: None)
+        Currently supported values are `num_rounds` (int, default: 1) and
+        `round_timeout` in seconds (float, default: None).
+    strategy : Optional[Strategy] (default: None)
+        An implementation of the abstract base class
+        `flwr.server.strategy.Strategy`. If no strategy is provided, then
+        `flwr.server.strategy.FedAvg` will be used.
+    client_manager : Optional[ClientManager] (default: None)
+        An implementation of the class `flwr.server.ClientManager`. If no
+        implementation is provided, then `flwr.server.SimpleClientManager`
+        will be used.
+    """
+
+    server: Optional[Server] = None
+    config: Optional[ServerConfig] = None
+    strategy: Optional[Strategy] = None
+    client_manager: Optional[ClientManager] = None

flwr/server/superlink/fleet/vce/backend/backend.py

@@ -33,8 +33,8 @@ class Backend(ABC):
         """Construct a backend."""
 
     @abstractmethod
-    async def build(self) -> None:
-        """Build backend asynchronously.
+    def build(self) -> None:
+        """Build backend.
 
         Different components need to be in place before workers in a backend are ready
         to accept jobs. When this method finishes executing, the backend should be fully
@@ -54,11 +54,11 @@ class Backend(ABC):
         """Report whether a backend worker is idle and can therefore run a ClientApp."""
 
     @abstractmethod
-    async def terminate(self) -> None:
+    def terminate(self) -> None:
         """Terminate backend."""
 
     @abstractmethod
-    async def process_message(
+    def process_message(
         self,
         app: Callable[[], ClientApp],
         message: Message,

flwr/server/superlink/fleet/vce/backend/raybackend.py

@@ -153,12 +153,12 @@ class RayBackend(Backend):
         """Report whether the pool has idle actors."""
         return self.pool.is_actor_available()
 
-    async def build(self) -> None:
+    def build(self) -> None:
         """Build pool of Ray actors that this backend will submit jobs to."""
-        await self.pool.add_actors_to_pool(self.pool.actors_capacity)
+        self.pool.add_actors_to_pool(self.pool.actors_capacity)
         log(DEBUG, "Constructed ActorPool with: %i actors", self.pool.num_actors)
 
-    async def process_message(
+    def process_message(
         self,
         app: Callable[[], ClientApp],
         message: Message,
@@ -172,17 +172,16 @@ class RayBackend(Backend):
 
         try:
             # Submit a task to the pool
-            future = await self.pool.submit(
+            future = self.pool.submit(
                 lambda a, a_fn, mssg, cid, state: a.run.remote(a_fn, mssg, cid, state),
                 (app, message, str(partition_id), context),
             )
 
-            await future
             # Fetch result
             (
                 out_mssg,
                 updated_context,
-            ) = await self.pool.fetch_result_and_return_actor_to_pool(future)
+            ) = self.pool.fetch_result_and_return_actor_to_pool(future)
 
             return out_mssg, updated_context
 
@@ -193,11 +192,11 @@ class RayBackend(Backend):
                 self.__class__.__name__,
             )
             # add actor back into pool
-            await self.pool.add_actor_back_to_pool(future)
+            self.pool.add_actor_back_to_pool(future)
             raise ex
 
-    async def terminate(self) -> None:
+    def terminate(self) -> None:
         """Terminate all actors in actor pool."""
-        await self.pool.terminate_all_actors()
+        self.pool.terminate_all_actors()
         ray.shutdown()
         log(DEBUG, "Terminated %s", self.__class__.__name__)