flwr-nightly 1.10.0.dev20240710__py3-none-any.whl → 1.10.0.dev20240712__py3-none-any.whl

flwr/cli/config_utils.py

@@ -108,6 +108,14 @@ def load(path: Optional[Path] = None) -> Optional[Dict[str, Any]]:
         return load_from_string(toml_file.read())
 
 
+def _validate_run_config(config_dict: Dict[str, Any], errors: List[str]) -> None:
+    for key, value in config_dict.items():
+        if isinstance(value, dict):
+            _validate_run_config(config_dict[key], errors)
+        elif not isinstance(value, str):
+            errors.append(f"Config value of key {key} is not of type `str`.")
+
+
 # pylint: disable=too-many-branches
 def validate_fields(config: Dict[str, Any]) -> Tuple[bool, List[str], List[str]]:
     """Validate pyproject.toml fields."""
@@ -133,6 +141,8 @@ def validate_fields(config: Dict[str, Any]) -> Tuple[bool, List[str], List[str]]
     else:
         if "publisher" not in config["flower"]:
             errors.append('Property "publisher" missing in [flower]')
+        if "config" in config["flower"]:
+            _validate_run_config(config["flower"]["config"], errors)
         if "components" not in config["flower"]:
             errors.append("Missing [flower.components] section")
         else:

flwr/cli/run/run.py

@@ -18,13 +18,14 @@ import sys
 from enum import Enum
 from logging import DEBUG
 from pathlib import Path
-from typing import Optional
+from typing import Dict, Optional
 
 import typer
 from typing_extensions import Annotated
 
 from flwr.cli import config_utils
 from flwr.cli.build import build
+from flwr.common.config import parse_config_args
 from flwr.common.constant import SUPEREXEC_DEFAULT_ADDRESS
 from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel
 from flwr.common.logger import log
@@ -58,15 +59,20 @@ def run(
         Optional[Path],
         typer.Option(help="Path of the Flower project to run"),
     ] = None,
+    config_overrides: Annotated[
+        Optional[str],
+        typer.Option(
+            "--config",
+            "-c",
+            help="Override configuration key-value pairs",
+        ),
+    ] = None,
 ) -> None:
     """Run Flower project."""
-    if use_superexec:
-        _start_superexec_run(directory)
-        return
-
     typer.secho("Loading project configuration... ", fg=typer.colors.BLUE)
 
-    config, errors, warnings = config_utils.load_and_validate()
+    pyproject_path = directory / "pyproject.toml" if directory else None
+    config, errors, warnings = config_utils.load_and_validate(path=pyproject_path)
 
     if config is None:
         typer.secho(
@@ -88,6 +94,12 @@ def run(
 
     typer.secho("Success", fg=typer.colors.GREEN)
 
+    if use_superexec:
+        _start_superexec_run(
+            parse_config_args(config_overrides, separator=","), directory
+        )
+        return
+
     server_app_ref = config["flower"]["components"]["serverapp"]
     client_app_ref = config["flower"]["components"]["clientapp"]
 
@@ -115,7 +127,9 @@ def run(
         )
 
 
-def _start_superexec_run(directory: Optional[Path]) -> None:
+def _start_superexec_run(
+    override_config: Dict[str, str], directory: Optional[Path]
+) -> None:
     def on_channel_state_change(channel_connectivity: str) -> None:
         """Log channel connectivity."""
         log(DEBUG, channel_connectivity)
@@ -132,6 +146,9 @@ def _start_superexec_run(directory: Optional[Path]) -> None:
 
     fab_path = build(directory)
 
-    req = StartRunRequest(fab_file=Path(fab_path).read_bytes())
+    req = StartRunRequest(
+        fab_file=Path(fab_path).read_bytes(),
+        override_config=override_config,
+    )
     res = stub.StartRun(req)
     typer.secho(f"🎊 Successfully started run {res.run_id}", fg=typer.colors.GREEN)

flwr/client/app.py

@@ -18,7 +18,8 @@ import signal
 import sys
 import time
 from dataclasses import dataclass
-from logging import DEBUG, ERROR, INFO, WARN
+from logging import ERROR, INFO, WARN
+from pathlib import Path
 from typing import Callable, ContextManager, Dict, Optional, Tuple, Type, Union
 
 from cryptography.hazmat.primitives.asymmetric import ec
@@ -159,6 +160,7 @@ def start_client(
     event(EventType.START_CLIENT_ENTER)
     _start_client_internal(
         server_address=server_address,
+        node_config={},
         load_client_app_fn=None,
         client_fn=client_fn,
         client=client,
@@ -180,6 +182,7 @@ def start_client(
 def _start_client_internal(
     *,
     server_address: str,
+    node_config: Dict[str, str],
     load_client_app_fn: Optional[Callable[[str, str], ClientApp]] = None,
     client_fn: Optional[ClientFnExt] = None,
     client: Optional[Client] = None,
@@ -192,7 +195,7 @@ def _start_client_internal(
     ] = None,
     max_retries: Optional[int] = None,
     max_wait_time: Optional[float] = None,
-    partition_id: Optional[int] = None,
+    flwr_dir: Optional[Path] = None,
 ) -> None:
     """Start a Flower client node which connects to a Flower server.
 
@@ -202,6 +205,8 @@ def _start_client_internal(
         The IPv4 or IPv6 address of the server. If the Flower
         server runs on the same machine on port 8080, then `server_address`
         would be `"[::]:8080"`.
+    node_config: Dict[str, str]
+        The configuration of the node.
     load_client_app_fn : Optional[Callable[[], ClientApp]] (default: None)
         A function that can be used to load a `ClientApp` instance.
     client_fn : Optional[ClientFnExt]
@@ -236,9 +241,8 @@ def _start_client_internal(
         The maximum duration before the client stops trying to
         connect to the server in case of connection error.
         If set to None, there is no limit to the total time.
-    partition_id: Optional[int] (default: None)
-        The data partition index associated with this node. Better suited for
-        prototyping purposes.
+    flwr_dir: Optional[Path] (default: None)
+        The fully resolved path containing installed Flower Apps.
     """
     if insecure is None:
         insecure = root_certificates is None
@@ -291,7 +295,7 @@ def _start_client_internal(
             log(WARN, "Connection attempt failed, retrying...")
         else:
             log(
-                DEBUG,
+                WARN,
                 "Connection attempt failed, retrying in %.2f seconds",
                 retry_state.actual_wait,
             )
@@ -315,8 +319,10 @@ def _start_client_internal(
         on_backoff=_on_backoff,
     )
 
-    node_state = NodeState(partition_id=partition_id)
-    run_info: Dict[int, Run] = {}
+    # NodeState gets initialized when the first connection is established
+    node_state: Optional[NodeState] = None
+
+    runs: Dict[int, Run] = {}
 
     while not app_state_tracker.interrupt:
         sleep_duration: int = 0
@@ -330,9 +336,33 @@ def _start_client_internal(
         ) as conn:
             receive, send, create_node, delete_node, get_run = conn
 
-            # Register node
-            if create_node is not None:
-                create_node()  # pylint: disable=not-callable
+            # Register node when connecting the first time
+            if node_state is None:
+                if create_node is None:
+                    if transport not in ["grpc-bidi", None]:
+                        raise NotImplementedError(
+                            "All transports except `grpc-bidi` require "
+                            "an implementation for `create_node()`.'"
+                        )
+                    # gRPC-bidi doesn't have the concept of node_id,
+                    # so we set it to -1
+                    node_state = NodeState(
+                        node_id=-1,
+                        node_config={},
+                        partition_id=None,
+                    )
+                else:
+                    # Call create_node fn to register node
+                    node_id: Optional[int] = (  # pylint: disable=assignment-from-none
+                        create_node()
+                    )  # pylint: disable=not-callable
+                    if node_id is None:
+                        raise ValueError("Node registration failed")
+                    node_state = NodeState(
+                        node_id=node_id,
+                        node_config=node_config,
+                        partition_id=None,
+                    )
 
             app_state_tracker.register_signal_handler()
             while not app_state_tracker.interrupt:
@@ -366,15 +396,17 @@ def _start_client_internal(
 
                     # Get run info
                     run_id = message.metadata.run_id
-                    if run_id not in run_info:
+                    if run_id not in runs:
                         if get_run is not None:
-                            run_info[run_id] = get_run(run_id)
+                            runs[run_id] = get_run(run_id)
                         # If get_run is None, i.e., in grpc-bidi mode
                         else:
-                            run_info[run_id] = Run(run_id, "", "", {})
+                            runs[run_id] = Run(run_id, "", "", {})
 
                     # Register context for this run
-                    node_state.register_context(run_id=run_id)
+                    node_state.register_context(
+                        run_id=run_id, run=runs[run_id], flwr_dir=flwr_dir
+                    )
 
                     # Retrieve context for this run
                     context = node_state.retrieve_context(run_id=run_id)
@@ -388,7 +420,7 @@ def _start_client_internal(
                     # Handle app loading and task message
                     try:
                         # Load ClientApp instance
-                        run: Run = run_info[run_id]
+                        run: Run = runs[run_id]
                         client_app: ClientApp = load_client_app_fn(
                             run.fab_id, run.fab_version
                         )
@@ -574,7 +606,7 @@ def _init_connection(transport: Optional[str], server_address: str) -> Tuple[
             Tuple[
                 Callable[[], Optional[Message]],
                 Callable[[Message], None],
-                Optional[Callable[[], None]],
+                Optional[Callable[[], Optional[int]]],
                 Optional[Callable[[], None]],
                 Optional[Callable[[int], Run]],
             ]

flwr/client/grpc_adapter_client/connection.py

@@ -44,7 +44,7 @@ def grpc_adapter(  # pylint: disable=R0913
     Tuple[
         Callable[[], Optional[Message]],
         Callable[[Message], None],
-        Optional[Callable[[], None]],
+        Optional[Callable[[], Optional[int]]],
         Optional[Callable[[], None]],
         Optional[Callable[[int], Run]],
     ]

flwr/client/grpc_client/connection.py

@@ -72,7 +72,7 @@ def grpc_connection(  # pylint: disable=R0913, R0915
     Tuple[
         Callable[[], Optional[Message]],
         Callable[[Message], None],
-        Optional[Callable[[], None]],
+        Optional[Callable[[], Optional[int]]],
         Optional[Callable[[], None]],
         Optional[Callable[[int], Run]],
     ]

flwr/client/grpc_rere_client/connection.py

@@ -79,7 +79,7 @@ def grpc_request_response(  # pylint: disable=R0913, R0914, R0915
     Tuple[
         Callable[[], Optional[Message]],
         Callable[[Message], None],
-        Optional[Callable[[], None]],
+        Optional[Callable[[], Optional[int]]],
         Optional[Callable[[], None]],
         Optional[Callable[[int], Run]],
     ]
@@ -176,7 +176,7 @@ def grpc_request_response(  # pylint: disable=R0913, R0914, R0915
         if not ping_stop_event.is_set():
             ping_stop_event.wait(next_interval)
 
-    def create_node() -> None:
+    def create_node() -> Optional[int]:
         """Set create_node."""
         # Call FleetAPI
         create_node_request = CreateNodeRequest(ping_interval=PING_DEFAULT_INTERVAL)
@@ -189,6 +189,7 @@ def grpc_request_response(  # pylint: disable=R0913, R0914, R0915
         nonlocal node, ping_thread
         node = cast(Node, create_node_response.node)
         ping_thread = start_ping_loop(ping, ping_stop_event)
+        return node.node_id
 
     def delete_node() -> None:
         """Set delete_node."""

flwr/client/node_state.py

@@ -15,30 +15,58 @@
 """Node state."""
 
 
-from typing import Any, Dict, Optional
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Optional
 
 from flwr.common import Context, RecordSet
+from flwr.common.config import get_fused_config
+from flwr.common.typing import Run
+
+
+@dataclass()
+class RunInfo:
+    """Contains the Context and initial run_config of a Run."""
+
+    context: Context
+    initial_run_config: Dict[str, str]
 
 
 class NodeState:
     """State of a node where client nodes execute runs."""
 
-    def __init__(self, partition_id: Optional[int]) -> None:
-        self._meta: Dict[str, Any] = {}  # holds metadata about the node
-        self.run_contexts: Dict[int, Context] = {}
+    def __init__(
+        self, node_id: int, node_config: Dict[str, str], partition_id: Optional[int]
+    ) -> None:
+        self.node_id = node_id
+        self.node_config = node_config
+        self.run_infos: Dict[int, RunInfo] = {}
         self._partition_id = partition_id
 
-    def register_context(self, run_id: int) -> None:
+    def register_context(
+        self,
+        run_id: int,
+        run: Optional[Run] = None,
+        flwr_dir: Optional[Path] = None,
+    ) -> None:
         """Register new run context for this node."""
-        if run_id not in self.run_contexts:
-            self.run_contexts[run_id] = Context(
-                state=RecordSet(), run_config={}, partition_id=self._partition_id
+        if run_id not in self.run_infos:
+            initial_run_config = get_fused_config(run, flwr_dir) if run else {}
+            self.run_infos[run_id] = RunInfo(
+                initial_run_config=initial_run_config,
+                context=Context(
+                    node_id=self.node_id,
+                    node_config=self.node_config,
+                    state=RecordSet(),
+                    run_config=initial_run_config.copy(),
+                    partition_id=self._partition_id,
+                ),
             )
 
     def retrieve_context(self, run_id: int) -> Context:
         """Get run context given a run_id."""
-        if run_id in self.run_contexts:
-            return self.run_contexts[run_id]
+        if run_id in self.run_infos:
+            return self.run_infos[run_id].context
 
         raise RuntimeError(
             f"Context for run_id={run_id} doesn't exist."
@@ -48,4 +76,9 @@ class NodeState:
 
     def update_context(self, run_id: int, context: Context) -> None:
         """Update run context."""
-        self.run_contexts[run_id] = context
+        if context.run_config != self.run_infos[run_id].initial_run_config:
+            raise ValueError(
+                "The `run_config` field of the `Context` object cannot be "
+                f"modified (run_id: {run_id})."
+            )
+        self.run_infos[run_id].context = context

flwr/client/node_state_tests.py

@@ -41,7 +41,7 @@ def test_multirun_in_node_state() -> None:
     expected_values = {0: "1", 1: "1" * 3, 2: "1" * 2, 3: "1", 5: "1"}
 
     # NodeState
-    node_state = NodeState(partition_id=None)
+    node_state = NodeState(node_id=0, node_config={}, partition_id=None)
 
     for task in tasks:
         run_id = task.run_id
@@ -59,7 +59,8 @@ def test_multirun_in_node_state() -> None:
         node_state.update_context(run_id=run_id, context=updated_state)
 
     # Verify values
-    for run_id, context in node_state.run_contexts.items():
+    for run_id, run_info in node_state.run_infos.items():
         assert (
-            context.state.configs_records["counter"]["count"] == expected_values[run_id]
+            run_info.context.state.configs_records["counter"]["count"]
+            == expected_values[run_id]
         )

flwr/client/rest_client/connection.py

@@ -90,7 +90,7 @@ def http_request_response(  # pylint: disable=,R0913, R0914, R0915
     Tuple[
         Callable[[], Optional[Message]],
         Callable[[Message], None],
-        Optional[Callable[[], None]],
+        Optional[Callable[[], Optional[int]]],
         Optional[Callable[[], None]],
         Optional[Callable[[int], Run]],
     ]
@@ -237,19 +237,20 @@ def http_request_response(  # pylint: disable=,R0913, R0914, R0915
         if not ping_stop_event.is_set():
             ping_stop_event.wait(next_interval)
 
-    def create_node() -> None:
+    def create_node() -> Optional[int]:
         """Set create_node."""
         req = CreateNodeRequest(ping_interval=PING_DEFAULT_INTERVAL)
 
         # Send the request
         res = _request(req, CreateNodeResponse, PATH_CREATE_NODE)
         if res is None:
-            return
+            return None
 
         # Remember the node and the ping-loop thread
         nonlocal node, ping_thread
         node = res.node
         ping_thread = start_ping_loop(ping, ping_stop_event)
+        return node.node_id
 
     def delete_node() -> None:
         """Set delete_node."""

flwr/client/supernode/app.py

@@ -29,7 +29,12 @@ from cryptography.hazmat.primitives.serialization import (
 
 from flwr.client.client_app import ClientApp, LoadClientAppError
 from flwr.common import EventType, event
-from flwr.common.config import get_flwr_dir, get_project_config, get_project_dir
+from flwr.common.config import (
+    get_flwr_dir,
+    get_project_config,
+    get_project_dir,
+    parse_config_args,
+)
 from flwr.common.constant import (
     TRANSPORT_TYPE_GRPC_ADAPTER,
     TRANSPORT_TYPE_GRPC_RERE,
@@ -67,7 +72,8 @@ def run_supernode() -> None:
         authentication_keys=authentication_keys,
         max_retries=args.max_retries,
         max_wait_time=args.max_wait_time,
-        partition_id=args.partition_id,
+        node_config=parse_config_args(args.node_config),
+        flwr_dir=get_flwr_dir(args.flwr_dir),
     )
 
     # Graceful shutdown
@@ -92,6 +98,7 @@ def run_client_app() -> None:
 
     _start_client_internal(
         server_address=args.superlink,
+        node_config=parse_config_args(args.node_config),
         load_client_app_fn=load_fn,
         transport=args.transport,
         root_certificates=root_certificates,
@@ -388,11 +395,11 @@ def _parse_args_common(parser: argparse.ArgumentParser) -> None:
         help="The SuperNode's public key (as a path str) to enable authentication.",
     )
     parser.add_argument(
-        "--partition-id",
-        type=int,
-        help="The data partition index associated with this SuperNode. Better suited "
-        "for prototyping purposes where a SuperNode might only load a fraction of an "
-        "artificially partitioned dataset (e.g. using `flwr-datasets`)",
+        "--node-config",
+        type=str,
+        help="A comma separated list of key/value pairs (separated by `=`) to "
+        "configure the SuperNode. "
+        "E.g. --node-config 'key1=\"value1\",partition-id=0,num-partitions=100'",
     )
 
 

flwr/common/config.py

@@ -121,16 +121,16 @@ def flatten_dict(raw_dict: Dict[str, Any], parent_key: str = "") -> Dict[str, st
 
 
 def parse_config_args(
-    config_overrides: Optional[str],
+    config: Optional[str],
     separator: str = ",",
 ) -> Dict[str, str]:
     """Parse separator separated list of key-value pairs separated by '='."""
     overrides: Dict[str, str] = {}
 
-    if config_overrides is None:
+    if config is None:
         return overrides
 
-    overrides_list = config_overrides.split(separator)
+    overrides_list = config.split(separator)
     if (
         len(overrides_list) == 1
         and "=" not in overrides_list

flwr/common/context.py

@@ -27,6 +27,11 @@ class Context:
 
     Parameters
     ----------
+    node_id : int
+        The ID that identifies the node.
+    node_config : Dict[str, str]
+        A config (key/value mapping) unique to the node and independent of the
+        `run_config`. This config persists across all runs this node participates in.
     state : RecordSet
         Holds records added by the entity in a given run and that will stay local.
         This means that the data it holds will never leave the system it's running from.
@@ -44,16 +49,22 @@ class Context:
         simulation or proto typing setups.
     """
 
+    node_id: int
+    node_config: Dict[str, str]
     state: RecordSet
-    partition_id: Optional[int]
     run_config: Dict[str, str]
+    partition_id: Optional[int]
 
-    def __init__(
+    def __init__(  # pylint: disable=too-many-arguments
         self,
+        node_id: int,
+        node_config: Dict[str, str],
         state: RecordSet,
         run_config: Dict[str, str],
         partition_id: Optional[int] = None,
     ) -> None:
+        self.node_id = node_id
+        self.node_config = node_config
         self.state = state
         self.run_config = run_config
         self.partition_id = partition_id

flwr/common/logger.py

@@ -197,6 +197,31 @@ def warn_deprecated_feature(name: str) -> None:
     )
 
 
+def warn_deprecated_feature_with_example(
+    deprecation_message: str, example_message: str, code_example: str
+) -> None:
+    """Warn if a feature is deprecated and show code example."""
+    log(
+        WARN,
+        """DEPRECATED FEATURE: %s
+
+            Check the following `FEATURE UPDATE` warning message for the preferred
+            new mechanism to use this feature in Flower.
+        """,
+        deprecation_message,
+    )
+    log(
+        WARN,
+        """FEATURE UPDATE: %s
+        ------------------------------------------------------------
+        %s
+        ------------------------------------------------------------
+        """,
+        example_message,
+        code_example,
+    )
+
+
 def warn_unsupported_feature(name: str) -> None:
     """Warn the user when they use an unsupported feature."""
     log(

flwr/server/__init__.py

@@ -28,6 +28,7 @@ from .run_serverapp import run_server_app as run_server_app
 from .server import Server as Server
 from .server_app import ServerApp as ServerApp
 from .server_config import ServerConfig as ServerConfig
+from .serverapp_components import ServerAppComponents as ServerAppComponents
 
 __all__ = [
     "ClientManager",
@@ -36,6 +37,7 @@ __all__ = [
     "LegacyContext",
     "Server",
     "ServerApp",
+    "ServerAppComponents",
     "ServerConfig",
     "SimpleClientManager",
     "run_server_app",

flwr/server/compat/legacy_context.py

@@ -52,4 +52,4 @@ class LegacyContext(Context):
         self.strategy = strategy
         self.client_manager = client_manager
         self.history = History()
-        super().__init__(state, run_config={})
+        super().__init__(node_id=0, node_config={}, state=state, run_config={})

flwr/server/run_serverapp.py

@@ -78,7 +78,9 @@ def run(
     server_app = _load()
 
     # Initialize Context
-    context = Context(state=RecordSet(), run_config=server_app_run_config)
+    context = Context(
+        node_id=0, node_config={}, state=RecordSet(), run_config=server_app_run_config
+    )
 
     # Call ServerApp
     server_app(driver=driver, context=context)