flwr-nightly 1.10.0.dev20240714__py3-none-any.whl → 1.10.0.dev20240716__py3-none-any.whl

flwr/cli/run/run.py

@@ -14,50 +14,33 @@
 # ==============================================================================
 """Flower command line interface `run` command."""
 
+import subprocess
 import sys
-from enum import Enum
 from logging import DEBUG
 from pathlib import Path
-from typing import Dict, Optional
+from typing import Any, Dict, Optional
 
 import typer
 from typing_extensions import Annotated
 
-from flwr.cli import config_utils
 from flwr.cli.build import build
+from flwr.cli.config_utils import load_and_validate
 from flwr.common.config import parse_config_args
-from flwr.common.constant import SUPEREXEC_DEFAULT_ADDRESS
 from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel
 from flwr.common.logger import log
 from flwr.proto.exec_pb2 import StartRunRequest  # pylint: disable=E0611
 from flwr.proto.exec_pb2_grpc import ExecStub
-from flwr.simulation.run_simulation import _run_simulation
-
-
-class Engine(str, Enum):
-    """Enum defining the engine to run on."""
-
-    SIMULATION = "simulation"
 
 
 # pylint: disable-next=too-many-locals
 def run(
-    engine: Annotated[
-        Optional[Engine],
-        typer.Option(
-            case_sensitive=False,
-            help="The engine to run FL with (currently only simulation is supported).",
-        ),
-    ] = None,
-    use_superexec: Annotated[
-        bool,
-        typer.Option(
-            case_sensitive=False, help="Use this flag to use the new SuperExec API"
-        ),
-    ] = False,
     directory: Annotated[
-        Optional[Path],
-        typer.Option(help="Path of the Flower project to run"),
+        Path,
+        typer.Argument(help="Path of the Flower project to run"),
+    ] = Path("."),
+    federation_name: Annotated[
+        Optional[str],
+        typer.Argument(help="Name of the federation to run the app on"),
     ] = None,
     config_overrides: Annotated[
         Optional[str],
@@ -72,7 +55,7 @@ def run(
     typer.secho("Loading project configuration... ", fg=typer.colors.BLUE)
 
     pyproject_path = directory / "pyproject.toml" if directory else None
-    config, errors, warnings = config_utils.load_and_validate(path=pyproject_path)
+    config, errors, warnings = load_and_validate(path=pyproject_path)
 
     if config is None:
         typer.secho(
@@ -94,50 +77,81 @@ def run(
 
     typer.secho("Success", fg=typer.colors.GREEN)
 
-    if use_superexec:
-        _start_superexec_run(
-            parse_config_args(config_overrides, separator=","), directory
-        )
-        return
-
-    server_app_ref = config["flower"]["components"]["serverapp"]
-    client_app_ref = config["flower"]["components"]["clientapp"]
-
-    if engine is None:
-        engine = config["flower"]["engine"]["name"]
+    federation_name = federation_name or config["tool"]["flwr"]["federations"].get(
+        "default"
+    )
 
-    if engine == Engine.SIMULATION:
-        num_supernodes = config["flower"]["engine"]["simulation"]["supernode"]["num"]
-        backend_config = config["flower"]["engine"]["simulation"].get(
-            "backend_config", None
+    if federation_name is None:
+        typer.secho(
+            "❌ No federation name was provided and the project's `pyproject.toml` "
+            "doesn't declare a default federation (with a SuperExec address or an "
+            "`options.num-supernodes` value).",
+            fg=typer.colors.RED,
+            bold=True,
         )
+        raise typer.Exit(code=1)
 
-        typer.secho("Starting run... ", fg=typer.colors.BLUE)
-        _run_simulation(
-            server_app_attr=server_app_ref,
-            client_app_attr=client_app_ref,
-            num_supernodes=num_supernodes,
-            backend_config=backend_config,
-        )
-    else:
+    # Validate the federation exists in the configuration
+    federation = config["tool"]["flwr"]["federations"].get(federation_name)
+    if federation is None:
+        available_feds = list(config["tool"]["flwr"]["federations"])
         typer.secho(
-            f"Engine '{engine}' is not yet supported in `flwr run`",
+            f"❌ There is no `{federation_name}` federation declared in the "
+            "`pyproject.toml`.\n The following federations were found:\n\n"
+            "\n".join(available_feds) + "\n\n",
             fg=typer.colors.RED,
             bold=True,
         )
+        raise typer.Exit(code=1)
+
+    if "address" in federation:
+        _run_with_superexec(federation, directory, config_overrides)
+    else:
+        _run_without_superexec(directory, federation, federation_name, config_overrides)
 
 
-def _start_superexec_run(
-    override_config: Dict[str, str], directory: Optional[Path]
+def _run_with_superexec(
+    federation: Dict[str, str],
+    directory: Optional[Path],
+    config_overrides: Optional[str],
 ) -> None:
+
     def on_channel_state_change(channel_connectivity: str) -> None:
         """Log channel connectivity."""
         log(DEBUG, channel_connectivity)
 
+    insecure_str = federation.get("insecure")
+    if root_certificates := federation.get("root-certificates"):
+        root_certificates_bytes = Path(root_certificates).read_bytes()
+        if insecure := bool(insecure_str):
+            typer.secho(
+                "❌ `root_certificates` were provided but the `insecure` parameter"
+                "is set to `True`.",
+                fg=typer.colors.RED,
+                bold=True,
+            )
+            raise typer.Exit(code=1)
+    else:
+        root_certificates_bytes = None
+        if insecure_str is None:
+            typer.secho(
+                "❌ To disable TLS, set `insecure = true` in `pyproject.toml`.",
+                fg=typer.colors.RED,
+                bold=True,
+            )
+            raise typer.Exit(code=1)
+        if not (insecure := bool(insecure_str)):
+            typer.secho(
+                "❌ No certificate were given yet `insecure` is set to `False`.",
+                fg=typer.colors.RED,
+                bold=True,
+            )
+            raise typer.Exit(code=1)
+
     channel = create_channel(
-        server_address=SUPEREXEC_DEFAULT_ADDRESS,
-        insecure=True,
-        root_certificates=None,
+        server_address=federation["address"],
+        insecure=insecure,
+        root_certificates=root_certificates_bytes,
         max_message_length=GRPC_MAX_MESSAGE_LENGTH,
         interceptors=None,
     )
@@ -148,7 +162,46 @@ def _start_superexec_run(
 
     req = StartRunRequest(
         fab_file=Path(fab_path).read_bytes(),
-        override_config=override_config,
+        override_config=parse_config_args(config_overrides, separator=","),
     )
     res = stub.StartRun(req)
     typer.secho(f"🎊 Successfully started run {res.run_id}", fg=typer.colors.GREEN)
+
+
+def _run_without_superexec(
+    app_path: Optional[Path],
+    federation: Dict[str, Any],
+    federation_name: str,
+    config_overrides: Optional[str],
+) -> None:
+    try:
+        num_supernodes = federation["options"]["num-supernodes"]
+    except KeyError as err:
+        typer.secho(
+            "❌ The project's `pyproject.toml` needs to declare the number of"
+            " SuperNodes in the simulation. To simulate 10 SuperNodes,"
+            " use the following notation:\n\n"
+            f"[tool.flwr.federations.{federation_name}]\n"
+            "options.num-supernodes = 10\n",
+            fg=typer.colors.RED,
+            bold=True,
+        )
+        raise typer.Exit(code=1) from err
+
+    command = [
+        "flower-simulation",
+        "--app",
+        f"{app_path}",
+        "--num-supernodes",
+        f"{num_supernodes}",
+    ]
+
+    if config_overrides:
+        command.extend(["--run-config", f"{config_overrides}"])
+
+    # Run the simulation
+    subprocess.run(
+        command,
+        check=True,
+        text=True,
+    )

flwr/client/app.py

@@ -195,7 +195,7 @@ def _start_client_internal(
     ] = None,
     max_retries: Optional[int] = None,
     max_wait_time: Optional[float] = None,
-    flwr_dir: Optional[Path] = None,
+    flwr_path: Optional[Path] = None,
 ) -> None:
     """Start a Flower client node which connects to a Flower server.
 
@@ -241,7 +241,7 @@ def _start_client_internal(
         The maximum duration before the client stops trying to
         connect to the server in case of connection error.
         If set to None, there is no limit to the total time.
-    flwr_dir: Optional[Path] (default: None)
+    flwr_path: Optional[Path] (default: None)
         The fully resolved path containing installed Flower Apps.
     """
     if insecure is None:
@@ -402,7 +402,7 @@ def _start_client_internal(
 
                     # Register context for this run
                     node_state.register_context(
-                        run_id=run_id, run=runs[run_id], flwr_dir=flwr_dir
+                        run_id=run_id, run=runs[run_id], flwr_path=flwr_path
                     )
 
                     # Retrieve context for this run

flwr/client/node_state.py

@@ -20,7 +20,7 @@ from pathlib import Path
 from typing import Dict, Optional
 
 from flwr.common import Context, RecordSet
-from flwr.common.config import get_fused_config
+from flwr.common.config import get_fused_config, get_fused_config_from_dir
 from flwr.common.typing import Run
 
 
@@ -48,11 +48,25 @@ class NodeState:
         self,
         run_id: int,
         run: Optional[Run] = None,
-        flwr_dir: Optional[Path] = None,
+        flwr_path: Optional[Path] = None,
+        app_dir: Optional[str] = None,
     ) -> None:
         """Register new run context for this node."""
         if run_id not in self.run_infos:
-            initial_run_config = get_fused_config(run, flwr_dir) if run else {}
+            initial_run_config = {}
+            if app_dir:
+                # Load from app directory
+                app_path = Path(app_dir)
+                if app_path.is_dir():
+                    override_config = run.override_config if run else {}
+                    initial_run_config = get_fused_config_from_dir(
+                        app_path, override_config
+                    )
+                else:
+                    raise ValueError("The specified `app_dir` must be a directory.")
+            else:
+                # Load from .fab
+                initial_run_config = get_fused_config(run, flwr_path) if run else {}
             self.run_infos[run_id] = RunInfo(
                 initial_run_config=initial_run_config,
                 context=Context(

flwr/client/supernode/app.py

@@ -60,7 +60,12 @@ def run_supernode() -> None:
     _warn_deprecated_server_arg(args)
 
     root_certificates = _get_certificates(args)
-    load_fn = _get_load_client_app_fn(args, multi_app=True)
+    load_fn = _get_load_client_app_fn(
+        default_app_ref=getattr(args, "client-app"),
+        dir_arg=args.dir,
+        flwr_dir_arg=args.flwr_dir,
+        multi_app=True,
+    )
     authentication_keys = _try_setup_client_authentication(args)
 
     _start_client_internal(
@@ -73,7 +78,7 @@ def run_supernode() -> None:
         max_retries=args.max_retries,
         max_wait_time=args.max_wait_time,
         node_config=parse_config_args(args.node_config),
-        flwr_dir=get_flwr_dir(args.flwr_dir),
+        flwr_path=get_flwr_dir(args.flwr_dir),
     )
 
     # Graceful shutdown
@@ -93,7 +98,11 @@ def run_client_app() -> None:
     _warn_deprecated_server_arg(args)
 
     root_certificates = _get_certificates(args)
-    load_fn = _get_load_client_app_fn(args, multi_app=False)
+    load_fn = _get_load_client_app_fn(
+        default_app_ref=getattr(args, "client-app"),
+        dir_arg=args.dir,
+        multi_app=False,
+    )
     authentication_keys = _try_setup_client_authentication(args)
 
     _start_client_internal(
@@ -166,7 +175,10 @@ def _get_certificates(args: argparse.Namespace) -> Optional[bytes]:
 
 
 def _get_load_client_app_fn(
-    args: argparse.Namespace, multi_app: bool
+    default_app_ref: str,
+    dir_arg: str,
+    multi_app: bool,
+    flwr_dir_arg: Optional[str] = None,
 ) -> Callable[[str, str], ClientApp]:
     """Get the load_client_app_fn function.
 
@@ -178,25 +190,24 @@ def _get_load_client_app_fn(
     loads a default ClientApp.
     """
     # Find the Flower directory containing Flower Apps (only for multi-app)
-    flwr_dir = Path("")
-    if "flwr_dir" in args:
-        if args.flwr_dir is None:
+    if not multi_app:
+        flwr_dir = Path("")
+    else:
+        if flwr_dir_arg is None:
             flwr_dir = get_flwr_dir()
         else:
-            flwr_dir = Path(args.flwr_dir).absolute()
+            flwr_dir = Path(flwr_dir_arg).absolute()
 
     inserted_path = None
 
-    default_app_ref: str = getattr(args, "client-app")
-
     if not multi_app:
         log(
             DEBUG,
             "Flower SuperNode will load and validate ClientApp `%s`",
-            getattr(args, "client-app"),
+            default_app_ref,
         )
         # Insert sys.path
-        dir_path = Path(args.dir).absolute()
+        dir_path = Path(dir_arg).absolute()
         sys.path.insert(0, str(dir_path))
         inserted_path = str(dir_path)
 
@@ -208,7 +219,7 @@ def _get_load_client_app_fn(
         # If multi-app feature is disabled
         if not multi_app:
             # Get sys path to be inserted
-            dir_path = Path(args.dir).absolute()
+            dir_path = Path(dir_arg).absolute()
 
             # Set app reference
             client_app_ref = default_app_ref
@@ -221,7 +232,7 @@ def _get_load_client_app_fn(
 
             log(WARN, "FAB ID is not provided; the default ClientApp will be loaded.")
             # Get sys path to be inserted
-            dir_path = Path(args.dir).absolute()
+            dir_path = Path(dir_arg).absolute()
 
             # Set app reference
             client_app_ref = default_app_ref
@@ -237,7 +248,7 @@ def _get_load_client_app_fn(
             dir_path = Path(project_dir).absolute()
 
             # Set app reference
-            client_app_ref = config["flower"]["components"]["clientapp"]
+            client_app_ref = config["tool"]["flwr"]["app"]["components"]["clientapp"]
 
         # Set sys.path
         nonlocal inserted_path

flwr/common/config.py

@@ -86,6 +86,18 @@ def _fuse_dicts(
     return fused_dict
 
 
+def get_fused_config_from_dir(
+    project_dir: Path, override_config: Dict[str, str]
+) -> Dict[str, str]:
+    """Merge the overrides from a given dict with the config from a Flower App."""
+    default_config = get_project_config(project_dir)["tool"]["flwr"]["app"].get(
+        "config", {}
+    )
+    flat_default_config = flatten_dict(default_config)
+
+    return _fuse_dicts(flat_default_config, override_config)
+
+
 def get_fused_config(run: Run, flwr_dir: Optional[Path]) -> Dict[str, str]:
     """Merge the overrides from a `Run` with the config from a FAB.
 
@@ -97,10 +109,7 @@ def get_fused_config(run: Run, flwr_dir: Optional[Path]) -> Dict[str, str]:
 
     project_dir = get_project_dir(run.fab_id, run.fab_version, flwr_dir)
 
-    default_config = get_project_config(project_dir)["flower"].get("config", {})
-    flat_default_config = flatten_dict(default_config)
-
-    return _fuse_dicts(flat_default_config, run.override_config)
+    return get_fused_config_from_dir(project_dir, run.override_config)
 
 
 def flatten_dict(raw_dict: Dict[str, Any], parent_key: str = "") -> Dict[str, str]:

flwr/server/run_serverapp.py

@@ -186,7 +186,7 @@ def run_server_app() -> None:  # pylint: disable=too-many-branches
         run_ = driver.run
         server_app_dir = str(get_project_dir(run_.fab_id, run_.fab_version, flwr_dir))
         config = get_project_config(server_app_dir)
-        server_app_attr = config["flower"]["components"]["serverapp"]
+        server_app_attr = config["tool"]["flwr"]["app"]["components"]["serverapp"]
         server_app_run_config = get_fused_config(run_, flwr_dir)
     else:
         # User provided `server-app`, but not `--run-id`

flwr/server/superlink/fleet/vce/vce_api.py

@@ -16,7 +16,6 @@
 
 
 import json
-import sys
 import threading
 import time
 import traceback
@@ -29,6 +28,7 @@ from typing import Callable, Dict, Optional
 
 from flwr.client.client_app import ClientApp, ClientAppException, LoadClientAppError
 from flwr.client.node_state import NodeState
+from flwr.client.supernode.app import _get_load_client_app_fn
 from flwr.common.constant import (
     NUM_PARTITIONS_KEY,
     PARTITION_ID_KEY,
@@ -37,8 +37,8 @@ from flwr.common.constant import (
 )
 from flwr.common.logger import log
 from flwr.common.message import Error
-from flwr.common.object_ref import load_app
 from flwr.common.serde import message_from_taskins, message_to_taskres
+from flwr.common.typing import Run
 from flwr.proto.task_pb2 import TaskIns, TaskRes  # pylint: disable=E0611
 from flwr.server.superlink.state import State, StateFactory
 
@@ -60,6 +60,31 @@ def _register_nodes(
     return nodes_mapping
 
 
+def _register_node_states(
+    nodes_mapping: NodeToPartitionMapping,
+    run: Run,
+    app_dir: Optional[str] = None,
+) -> Dict[int, NodeState]:
+    """Create NodeState objects and pre-register the context for the run."""
+    node_states: Dict[int, NodeState] = {}
+    num_partitions = len(set(nodes_mapping.values()))
+    for node_id, partition_id in nodes_mapping.items():
+        node_states[node_id] = NodeState(
+            node_id=node_id,
+            node_config={
+                PARTITION_ID_KEY: str(partition_id),
+                NUM_PARTITIONS_KEY: str(num_partitions),
+            },
+        )
+
+        # Pre-register Context objects
+        node_states[node_id].register_context(
+            run_id=run.run_id, run=run, app_dir=app_dir
+        )
+
+    return node_states
+
+
 # pylint: disable=too-many-arguments,too-many-locals
 def worker(
     app_fn: Callable[[], ClientApp],
@@ -78,8 +103,7 @@ def worker(
             task_ins: TaskIns = taskins_queue.get(timeout=1.0)
             node_id = task_ins.task.consumer.node_id
 
-            # Register and retrieve context
-            node_states[node_id].register_context(run_id=task_ins.run_id)
+            # Retrieve context
             context = node_states[node_id].retrieve_context(run_id=task_ins.run_id)
 
             # Convert TaskIns to Message
@@ -151,7 +175,7 @@ def put_taskres_into_state(
             pass
 
 
-def run(
+def run_api(
     app_fn: Callable[[], ClientApp],
     backend_fn: Callable[[], Backend],
     nodes_mapping: NodeToPartitionMapping,
@@ -236,7 +260,10 @@ def start_vce(
     backend_name: str,
     backend_config_json_stream: str,
     app_dir: str,
+    is_app: bool,
     f_stop: threading.Event,
+    run: Run,
+    flwr_dir: Optional[str] = None,
     client_app: Optional[ClientApp] = None,
     client_app_attr: Optional[str] = None,
     num_supernodes: Optional[int] = None,
@@ -287,17 +314,9 @@ def start_vce(
         )
 
     # Construct mapping of NodeStates
-    node_states: Dict[int, NodeState] = {}
-    # Number of unique partitions
-    num_partitions = len(set(nodes_mapping.values()))
-    for node_id, partition_id in nodes_mapping.items():
-        node_states[node_id] = NodeState(
-            node_id=node_id,
-            node_config={
-                PARTITION_ID_KEY: str(partition_id),
-                NUM_PARTITIONS_KEY: str(num_partitions),
-            },
-        )
+    node_states = _register_node_states(
+        nodes_mapping=nodes_mapping, run=run, app_dir=app_dir if is_app else None
+    )
 
     # Load backend config
     log(DEBUG, "Supported backends: %s", list(supported_backends.keys()))
@@ -326,16 +345,12 @@ def start_vce(
     def _load() -> ClientApp:
 
         if client_app_attr:
-
-            if app_dir is not None:
-                sys.path.insert(0, app_dir)
-
-            app: ClientApp = load_app(client_app_attr, LoadClientAppError, app_dir)
-
-            if not isinstance(app, ClientApp):
-                raise LoadClientAppError(
-                    f"Attribute {client_app_attr} is not of type {ClientApp}",
-                ) from None
+            app = _get_load_client_app_fn(
+                default_app_ref=client_app_attr,
+                dir_arg=app_dir,
+                flwr_dir_arg=flwr_dir,
+                multi_app=True,
+            )(run.fab_id, run.fab_version)
 
         if client_app:
             app = client_app
@@ -345,10 +360,19 @@ def start_vce(
 
     try:
         # Test if ClientApp can be loaded
-        _ = app_fn()
+        client_app = app_fn()
+
+        # Cache `ClientApp`
+        if client_app_attr:
+            # Now wrap the loaded ClientApp in a dummy function
+            # this prevent unnecesary low-level loading of ClientApp
+            def _load_client_app() -> ClientApp:
+                return client_app
+
+            app_fn = _load_client_app
 
         # Run main simulation loop
-        run(
+        run_api(
             app_fn,
             backend_fn,
             nodes_mapping,