flwr-nightly 1.10.0.dev20240707__py3-none-any.whl → 1.10.0.dev20240722__py3-none-any.whl

flwr/cli/run/run.py

@@ -14,59 +14,49 @@
 # ==============================================================================
 """Flower command line interface `run` command."""
 
+import subprocess
 import sys
-from enum import Enum
 from logging import DEBUG
 from pathlib import Path
-from typing import Optional
+from typing import Any, Dict, List, Optional
 
 import typer
 from typing_extensions import Annotated
 
-from flwr.cli import config_utils
 from flwr.cli.build import build
-from flwr.common.constant import SUPEREXEC_DEFAULT_ADDRESS
+from flwr.cli.config_utils import load_and_validate
+from flwr.common.config import parse_config_args
 from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel
 from flwr.common.logger import log
+from flwr.common.serde import user_config_to_proto
 from flwr.proto.exec_pb2 import StartRunRequest  # pylint: disable=E0611
 from flwr.proto.exec_pb2_grpc import ExecStub
-from flwr.simulation.run_simulation import _run_simulation
-
-
-class Engine(str, Enum):
-    """Enum defining the engine to run on."""
-
-    SIMULATION = "simulation"
 
 
 # pylint: disable-next=too-many-locals
 def run(
-    engine: Annotated[
-        Optional[Engine],
-        typer.Option(
-            case_sensitive=False,
-            help="The engine to run FL with (currently only simulation is supported).",
-        ),
+    directory: Annotated[
+        Path,
+        typer.Argument(help="Path of the Flower project to run"),
+    ] = Path("."),
+    federation_name: Annotated[
+        Optional[str],
+        typer.Argument(help="Name of the federation to run the app on"),
     ] = None,
-    use_superexec: Annotated[
-        bool,
+    config_overrides: Annotated[
+        Optional[List[str]],
         typer.Option(
-            case_sensitive=False, help="Use this flag to use the new SuperExec API"
+            "--run-config",
+            "-c",
+            help="Override configuration key-value pairs",
         ),
-    ] = False,
-    directory: Annotated[
-        Optional[Path],
-        typer.Option(help="Path of the Flower project to run"),
     ] = None,
 ) -> None:
     """Run Flower project."""
-    if use_superexec:
-        _start_superexec_run(directory)
-        return
-
     typer.secho("Loading project configuration... ", fg=typer.colors.BLUE)
 
-    config, errors, warnings = config_utils.load_and_validate()
+    pyproject_path = directory / "pyproject.toml" if directory else None
+    config, errors, warnings = load_and_validate(path=pyproject_path)
 
     if config is None:
         typer.secho(
@@ -88,42 +78,83 @@ def run(
 
     typer.secho("Success", fg=typer.colors.GREEN)
 
-    server_app_ref = config["flower"]["components"]["serverapp"]
-    client_app_ref = config["flower"]["components"]["clientapp"]
-
-    if engine is None:
-        engine = config["flower"]["engine"]["name"]
-
-    if engine == Engine.SIMULATION:
-        num_supernodes = config["flower"]["engine"]["simulation"]["supernode"]["num"]
-        backend_config = config["flower"]["engine"]["simulation"].get(
-            "backend_config", None
-        )
+    federation_name = federation_name or config["tool"]["flwr"]["federations"].get(
+        "default"
+    )
 
-        typer.secho("Starting run... ", fg=typer.colors.BLUE)
-        _run_simulation(
-            server_app_attr=server_app_ref,
-            client_app_attr=client_app_ref,
-            num_supernodes=num_supernodes,
-            backend_config=backend_config,
+    if federation_name is None:
+        typer.secho(
+            "❌ No federation name was provided and the project's `pyproject.toml` "
+            "doesn't declare a default federation (with a SuperExec address or an "
+            "`options.num-supernodes` value).",
+            fg=typer.colors.RED,
+            bold=True,
         )
-    else:
+        raise typer.Exit(code=1)
+
+    # Validate the federation exists in the configuration
+    federation = config["tool"]["flwr"]["federations"].get(federation_name)
+    if federation is None:
+        available_feds = {
+            fed for fed in config["tool"]["flwr"]["federations"] if fed != "default"
+        }
         typer.secho(
-            f"Engine '{engine}' is not yet supported in `flwr run`",
+            f"❌ There is no `{federation_name}` federation declared in the "
+            "`pyproject.toml`.\n The following federations were found:\n\n"
+            + "\n".join(available_feds),
             fg=typer.colors.RED,
             bold=True,
         )
+        raise typer.Exit(code=1)
 
+    if "address" in federation:
+        _run_with_superexec(federation, directory, config_overrides)
+    else:
+        _run_without_superexec(directory, federation, federation_name, config_overrides)
+
+
+def _run_with_superexec(
+    federation: Dict[str, str],
+    directory: Optional[Path],
+    config_overrides: Optional[List[str]],
+) -> None:
 
-def _start_superexec_run(directory: Optional[Path]) -> None:
     def on_channel_state_change(channel_connectivity: str) -> None:
         """Log channel connectivity."""
         log(DEBUG, channel_connectivity)
 
+    insecure_str = federation.get("insecure")
+    if root_certificates := federation.get("root-certificates"):
+        root_certificates_bytes = Path(root_certificates).read_bytes()
+        if insecure := bool(insecure_str):
+            typer.secho(
+                "❌ `root_certificates` were provided but the `insecure` parameter"
+                "is set to `True`.",
+                fg=typer.colors.RED,
+                bold=True,
+            )
+            raise typer.Exit(code=1)
+    else:
+        root_certificates_bytes = None
+        if insecure_str is None:
+            typer.secho(
+                "❌ To disable TLS, set `insecure = true` in `pyproject.toml`.",
+                fg=typer.colors.RED,
+                bold=True,
+            )
+            raise typer.Exit(code=1)
+        if not (insecure := bool(insecure_str)):
+            typer.secho(
+                "❌ No certificate were given yet `insecure` is set to `False`.",
+                fg=typer.colors.RED,
+                bold=True,
+            )
+            raise typer.Exit(code=1)
+
     channel = create_channel(
-        server_address=SUPEREXEC_DEFAULT_ADDRESS,
-        insecure=True,
-        root_certificates=None,
+        server_address=federation["address"],
+        insecure=insecure,
+        root_certificates=root_certificates_bytes,
         max_message_length=GRPC_MAX_MESSAGE_LENGTH,
         interceptors=None,
     )
@@ -132,6 +163,50 @@ def _start_superexec_run(directory: Optional[Path]) -> None:
 
     fab_path = build(directory)
 
-    req = StartRunRequest(fab_file=Path(fab_path).read_bytes())
+    req = StartRunRequest(
+        fab_file=Path(fab_path).read_bytes(),
+        override_config=user_config_to_proto(
+            parse_config_args(config_overrides, separator=",")
+        ),
+    )
     res = stub.StartRun(req)
     typer.secho(f"🎊 Successfully started run {res.run_id}", fg=typer.colors.GREEN)
+
+
+def _run_without_superexec(
+    app_path: Optional[Path],
+    federation: Dict[str, Any],
+    federation_name: str,
+    config_overrides: Optional[List[str]],
+) -> None:
+    try:
+        num_supernodes = federation["options"]["num-supernodes"]
+    except KeyError as err:
+        typer.secho(
+            "❌ The project's `pyproject.toml` needs to declare the number of"
+            " SuperNodes in the simulation. To simulate 10 SuperNodes,"
+            " use the following notation:\n\n"
+            f"[tool.flwr.federations.{federation_name}]\n"
+            "options.num-supernodes = 10\n",
+            fg=typer.colors.RED,
+            bold=True,
+        )
+        raise typer.Exit(code=1) from err
+
+    command = [
+        "flower-simulation",
+        "--app",
+        f"{app_path}",
+        "--num-supernodes",
+        f"{num_supernodes}",
+    ]
+
+    if config_overrides:
+        command.extend(["--run-config", f"{','.join(config_overrides)}"])
+
+    # Run the simulation
+    subprocess.run(
+        command,
+        check=True,
+        text=True,
+    )

flwr/client/app.py

@@ -18,7 +18,8 @@ import signal
 import sys
 import time
 from dataclasses import dataclass
-from logging import DEBUG, ERROR, INFO, WARN
+from logging import ERROR, INFO, WARN
+from pathlib import Path
 from typing import Callable, ContextManager, Dict, Optional, Tuple, Type, Union
 
 from cryptography.hazmat.primitives.asymmetric import ec
@@ -27,7 +28,7 @@ from grpc import RpcError
 from flwr.client.client import Client
 from flwr.client.client_app import ClientApp, LoadClientAppError
 from flwr.client.typing import ClientFnExt
-from flwr.common import GRPC_MAX_MESSAGE_LENGTH, EventType, Message, event
+from flwr.common import GRPC_MAX_MESSAGE_LENGTH, Context, EventType, Message, event
 from flwr.common.address import parse_address
 from flwr.common.constant import (
     MISSING_EXTRA_REST,
@@ -41,6 +42,7 @@ from flwr.common.constant import (
 from flwr.common.logger import log, warn_deprecated_feature
 from flwr.common.message import Error
 from flwr.common.retry_invoker import RetryInvoker, RetryState, exponential
+from flwr.common.typing import Run, UserConfig
 
 from .grpc_adapter_client.connection import grpc_adapter
 from .grpc_client.connection import grpc_connection
@@ -136,8 +138,8 @@ def start_client(
 
     Starting an SSL-enabled gRPC client using system certificates:
 
-    >>> def client_fn(node_id: int, partition_id: Optional[int]):
-    >>>     return FlowerClient()
+    >>> def client_fn(context: Context):
+    >>>     return FlowerClient().to_client()
     >>>
     >>> start_client(
     >>>     server_address=localhost:8080,
@@ -158,6 +160,7 @@ def start_client(
     event(EventType.START_CLIENT_ENTER)
     _start_client_internal(
         server_address=server_address,
+        node_config={},
         load_client_app_fn=None,
         client_fn=client_fn,
         client=client,
@@ -179,6 +182,7 @@ def start_client(
 def _start_client_internal(
     *,
     server_address: str,
+    node_config: UserConfig,
     load_client_app_fn: Optional[Callable[[str, str], ClientApp]] = None,
     client_fn: Optional[ClientFnExt] = None,
     client: Optional[Client] = None,
@@ -191,7 +195,7 @@ def _start_client_internal(
     ] = None,
     max_retries: Optional[int] = None,
     max_wait_time: Optional[float] = None,
-    partition_id: Optional[int] = None,
+    flwr_path: Optional[Path] = None,
 ) -> None:
     """Start a Flower client node which connects to a Flower server.
 
@@ -201,6 +205,8 @@ def _start_client_internal(
         The IPv4 or IPv6 address of the server. If the Flower
         server runs on the same machine on port 8080, then `server_address`
         would be `"[::]:8080"`.
+    node_config: UserConfig
+        The configuration of the node.
     load_client_app_fn : Optional[Callable[[], ClientApp]] (default: None)
         A function that can be used to load a `ClientApp` instance.
     client_fn : Optional[ClientFnExt]
@@ -235,9 +241,8 @@ def _start_client_internal(
         The maximum duration before the client stops trying to
         connect to the server in case of connection error.
         If set to None, there is no limit to the total time.
-    partitioni_id: Optional[int] (default: None)
-        The data partition index associated with this node. Better suited for
-        prototyping purposes.
+    flwr_path: Optional[Path] (default: None)
+        The fully resolved path containing installed Flower Apps.
     """
     if insecure is None:
         insecure = root_certificates is None
@@ -248,8 +253,7 @@ def _start_client_internal(
         if client_fn is None:
             # Wrap `Client` instance in `client_fn`
             def single_client_factory(
-                node_id: int,  # pylint: disable=unused-argument
-                partition_id: Optional[int],  # pylint: disable=unused-argument
+                context: Context,  # pylint: disable=unused-argument
             ) -> Client:
                 if client is None:  # Added this to keep mypy happy
                     raise ValueError(
@@ -290,7 +294,7 @@ def _start_client_internal(
             log(WARN, "Connection attempt failed, retrying...")
         else:
             log(
-                DEBUG,
+                WARN,
                 "Connection attempt failed, retrying in %.2f seconds",
                 retry_state.actual_wait,
             )
@@ -314,9 +318,10 @@ def _start_client_internal(
         on_backoff=_on_backoff,
     )
 
-    node_state = NodeState(partition_id=partition_id)
-    # run_id -> (fab_id, fab_version)
-    run_info: Dict[int, Tuple[str, str]] = {}
+    # NodeState gets initialized when the first connection is established
+    node_state: Optional[NodeState] = None
+
+    runs: Dict[int, Run] = {}
 
     while not app_state_tracker.interrupt:
         sleep_duration: int = 0
@@ -330,9 +335,31 @@ def _start_client_internal(
         ) as conn:
             receive, send, create_node, delete_node, get_run = conn
 
-            # Register node
-            if create_node is not None:
-                create_node()  # pylint: disable=not-callable
+            # Register node when connecting the first time
+            if node_state is None:
+                if create_node is None:
+                    if transport not in ["grpc-bidi", None]:
+                        raise NotImplementedError(
+                            "All transports except `grpc-bidi` require "
+                            "an implementation for `create_node()`.'"
+                        )
+                    # gRPC-bidi doesn't have the concept of node_id,
+                    # so we set it to -1
+                    node_state = NodeState(
+                        node_id=-1,
+                        node_config={},
+                    )
+                else:
+                    # Call create_node fn to register node
+                    node_id: Optional[int] = (  # pylint: disable=assignment-from-none
+                        create_node()
+                    )  # pylint: disable=not-callable
+                    if node_id is None:
+                        raise ValueError("Node registration failed")
+                    node_state = NodeState(
+                        node_id=node_id,
+                        node_config=node_config,
+                    )
 
             app_state_tracker.register_signal_handler()
             while not app_state_tracker.interrupt:
@@ -366,15 +393,17 @@ def _start_client_internal(
 
                     # Get run info
                     run_id = message.metadata.run_id
-                    if run_id not in run_info:
+                    if run_id not in runs:
                         if get_run is not None:
-                            run_info[run_id] = get_run(run_id)
+                            runs[run_id] = get_run(run_id)
                         # If get_run is None, i.e., in grpc-bidi mode
                         else:
-                            run_info[run_id] = ("", "")
+                            runs[run_id] = Run(run_id, "", "", {})
 
                     # Register context for this run
-                    node_state.register_context(run_id=run_id)
+                    node_state.register_context(
+                        run_id=run_id, run=runs[run_id], flwr_path=flwr_path
+                    )
 
                     # Retrieve context for this run
                     context = node_state.retrieve_context(run_id=run_id)
@@ -388,7 +417,10 @@ def _start_client_internal(
                     # Handle app loading and task message
                     try:
                         # Load ClientApp instance
-                        client_app: ClientApp = load_client_app_fn(*run_info[run_id])
+                        run: Run = runs[run_id]
+                        client_app: ClientApp = load_client_app_fn(
+                            run.fab_id, run.fab_version
+                        )
 
                         # Execute ClientApp
                         reply_message = client_app(message=message, context=context)
@@ -571,9 +603,9 @@ def _init_connection(transport: Optional[str], server_address: str) -> Tuple[
             Tuple[
                 Callable[[], Optional[Message]],
                 Callable[[Message], None],
+                Optional[Callable[[], Optional[int]]],
                 Optional[Callable[[], None]],
-                Optional[Callable[[], None]],
-                Optional[Callable[[int], Tuple[str, str]]],
+                Optional[Callable[[int], Run]],
             ]
         ],
     ],

flwr/client/client_app.py

@@ -30,21 +30,41 @@ from flwr.common.logger import warn_deprecated_feature, warn_preview_feature
 from .typing import ClientAppCallable
 
 
+def _alert_erroneous_client_fn() -> None:
+    raise ValueError(
+        "A `ClientApp` cannot make use of a `client_fn` that does "
+        "not have a signature in the form: `def client_fn(context: "
+        "Context)`. You can import the `Context` like this: "
+        "`from flwr.common import Context`"
+    )
+
+
 def _inspect_maybe_adapt_client_fn_signature(client_fn: ClientFnExt) -> ClientFnExt:
     client_fn_args = inspect.signature(client_fn).parameters
+    first_arg = list(client_fn_args.keys())[0]
+
+    if len(client_fn_args) != 1:
+        _alert_erroneous_client_fn()
+
+    first_arg_type = client_fn_args[first_arg].annotation
 
-    if not all(key in client_fn_args for key in ["node_id", "partition_id"]):
+    if first_arg_type is str or first_arg == "cid":
+        # Warn previous signature for `client_fn` seems to be used
         warn_deprecated_feature(
-            "`client_fn` now expects a signature `def client_fn(node_id: int, "
-            "partition_id: Optional[int])`.\nYou provided `client_fn` with signature: "
-            f"{dict(client_fn_args.items())}"
+            "`client_fn` now expects a signature `def client_fn(context: Context)`."
+            "The provided `client_fn` has signature: "
+            f"{dict(client_fn_args.items())}. You can import the `Context` like this:"
+            " `from flwr.common import Context`"
         )
 
         # Wrap depcreated client_fn inside a function with the expected signature
         def adaptor_fn(
-            node_id: int, partition_id: Optional[int]  # pylint: disable=unused-argument
-        ) -> Client:
-            return client_fn(str(partition_id))  # type: ignore
+            context: Context,
+        ) -> Client:  # pylint: disable=unused-argument
+            # if patition-id is defined, pass it. Else pass node_id that should
+            # always be defined during Context init.
+            cid = context.node_config.get("partition-id", context.node_id)
+            return client_fn(str(cid))  # type: ignore
 
         return adaptor_fn
 
@@ -71,7 +91,7 @@ class ClientApp:
     >>> class FlowerClient(NumPyClient):
     >>>     # ...
     >>>
-    >>> def client_fn(node_id: int, partition_id: Optional[int]):
+    >>> def client_fn(context: Context):
     >>>    return FlowerClient().to_client()
     >>>
     >>> app = ClientApp(client_fn)

flwr/client/grpc_adapter_client/connection.py

@@ -27,6 +27,7 @@ from flwr.common import GRPC_MAX_MESSAGE_LENGTH
 from flwr.common.logger import log
 from flwr.common.message import Message
 from flwr.common.retry_invoker import RetryInvoker
+from flwr.common.typing import Run
 
 
 @contextmanager
@@ -43,9 +44,9 @@ def grpc_adapter(  # pylint: disable=R0913
     Tuple[
         Callable[[], Optional[Message]],
         Callable[[Message], None],
+        Optional[Callable[[], Optional[int]]],
         Optional[Callable[[], None]],
-        Optional[Callable[[], None]],
-        Optional[Callable[[int], Tuple[str, str]]],
+        Optional[Callable[[int], Run]],
     ]
 ]:
     """Primitives for request/response-based interaction with a server via GrpcAdapter.

flwr/client/grpc_client/connection.py

@@ -38,6 +38,7 @@ from flwr.common.constant import MessageType, MessageTypeLegacy
 from flwr.common.grpc import create_channel
 from flwr.common.logger import log
 from flwr.common.retry_invoker import RetryInvoker
+from flwr.common.typing import Run
 from flwr.proto.transport_pb2 import (  # pylint: disable=E0611
     ClientMessage,
     Reason,
@@ -71,9 +72,9 @@ def grpc_connection(  # pylint: disable=R0913, R0915
     Tuple[
         Callable[[], Optional[Message]],
         Callable[[Message], None],
+        Optional[Callable[[], Optional[int]]],
         Optional[Callable[[], None]],
-        Optional[Callable[[], None]],
-        Optional[Callable[[int], Tuple[str, str]]],
+        Optional[Callable[[int], Run]],
     ]
 ]:
     """Establish a gRPC connection to a gRPC server.

flwr/client/grpc_rere_client/connection.py

@@ -40,7 +40,12 @@ from flwr.common.grpc import create_channel
 from flwr.common.logger import log
 from flwr.common.message import Message, Metadata
 from flwr.common.retry_invoker import RetryInvoker
-from flwr.common.serde import message_from_taskins, message_to_taskres
+from flwr.common.serde import (
+    message_from_taskins,
+    message_to_taskres,
+    user_config_from_proto,
+)
+from flwr.common.typing import Run
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     DeleteNodeRequest,
@@ -78,9 +83,9 @@ def grpc_request_response(  # pylint: disable=R0913, R0914, R0915
     Tuple[
         Callable[[], Optional[Message]],
         Callable[[Message], None],
+        Optional[Callable[[], Optional[int]]],
         Optional[Callable[[], None]],
-        Optional[Callable[[], None]],
-        Optional[Callable[[int], Tuple[str, str]]],
+        Optional[Callable[[int], Run]],
     ]
 ]:
     """Primitives for request/response-based interaction with a server.
@@ -175,7 +180,7 @@ def grpc_request_response(  # pylint: disable=R0913, R0914, R0915
         if not ping_stop_event.is_set():
             ping_stop_event.wait(next_interval)
 
-    def create_node() -> None:
+    def create_node() -> Optional[int]:
         """Set create_node."""
         # Call FleetAPI
         create_node_request = CreateNodeRequest(ping_interval=PING_DEFAULT_INTERVAL)
@@ -188,6 +193,7 @@ def grpc_request_response(  # pylint: disable=R0913, R0914, R0915
         nonlocal node, ping_thread
         node = cast(Node, create_node_response.node)
         ping_thread = start_ping_loop(ping, ping_stop_event)
+        return node.node_id
 
     def delete_node() -> None:
         """Set delete_node."""
@@ -266,7 +272,7 @@ def grpc_request_response(  # pylint: disable=R0913, R0914, R0915
         # Cleanup
         metadata = None
 
-    def get_run(run_id: int) -> Tuple[str, str]:
+    def get_run(run_id: int) -> Run:
         # Call FleetAPI
         get_run_request = GetRunRequest(run_id=run_id)
         get_run_response: GetRunResponse = retry_invoker.invoke(
@@ -275,7 +281,12 @@ def grpc_request_response(  # pylint: disable=R0913, R0914, R0915
         )
 
         # Return fab_id and fab_version
-        return get_run_response.run.fab_id, get_run_response.run.fab_version
+        return Run(
+            run_id,
+            get_run_response.run.fab_id,
+            get_run_response.run.fab_version,
+            user_config_from_proto(get_run_response.run.override_config),
+        )
 
     try:
         # Yield methods

flwr/client/message_handler/message_handler.py

@@ -92,7 +92,7 @@ def handle_legacy_message_from_msgtype(
     client_fn: ClientFnExt, message: Message, context: Context
 ) -> Message:
     """Handle legacy message in the inner most mod."""
-    client = client_fn(message.metadata.dst_node_id, context.partition_id)
+    client = client_fn(context)
 
     # Check if NumPyClient is returend
     if isinstance(client, NumPyClient):