flwr-nightly 1.10.0.dev20240715__py3-none-any.whl → 1.10.0.dev20240717__py3-none-any.whl

flwr/client/app.py

@@ -195,7 +195,7 @@ def _start_client_internal(
     ] = None,
     max_retries: Optional[int] = None,
     max_wait_time: Optional[float] = None,
-    flwr_dir: Optional[Path] = None,
+    flwr_path: Optional[Path] = None,
 ) -> None:
     """Start a Flower client node which connects to a Flower server.
 
@@ -241,7 +241,7 @@ def _start_client_internal(
         The maximum duration before the client stops trying to
         connect to the server in case of connection error.
         If set to None, there is no limit to the total time.
-    flwr_dir: Optional[Path] (default: None)
+    flwr_path: Optional[Path] (default: None)
         The fully resolved path containing installed Flower Apps.
     """
     if insecure is None:
@@ -402,7 +402,7 @@ def _start_client_internal(
 
                     # Register context for this run
                     node_state.register_context(
-                        run_id=run_id, run=runs[run_id], flwr_dir=flwr_dir
+                        run_id=run_id, run=runs[run_id], flwr_path=flwr_path
                     )
 
                     # Retrieve context for this run

flwr/client/node_state.py

@@ -20,7 +20,7 @@ from pathlib import Path
 from typing import Dict, Optional
 
 from flwr.common import Context, RecordSet
-from flwr.common.config import get_fused_config
+from flwr.common.config import get_fused_config, get_fused_config_from_dir
 from flwr.common.typing import Run
 
 
@@ -48,11 +48,25 @@ class NodeState:
         self,
         run_id: int,
         run: Optional[Run] = None,
-        flwr_dir: Optional[Path] = None,
+        flwr_path: Optional[Path] = None,
+        app_dir: Optional[str] = None,
     ) -> None:
         """Register new run context for this node."""
         if run_id not in self.run_infos:
-            initial_run_config = get_fused_config(run, flwr_dir) if run else {}
+            initial_run_config = {}
+            if app_dir:
+                # Load from app directory
+                app_path = Path(app_dir)
+                if app_path.is_dir():
+                    override_config = run.override_config if run else {}
+                    initial_run_config = get_fused_config_from_dir(
+                        app_path, override_config
+                    )
+                else:
+                    raise ValueError("The specified `app_dir` must be a directory.")
+            else:
+                # Load from .fab
+                initial_run_config = get_fused_config(run, flwr_path) if run else {}
             self.run_infos[run_id] = RunInfo(
                 initial_run_config=initial_run_config,
                 context=Context(

flwr/client/supernode/app.py

@@ -77,8 +77,8 @@ def run_supernode() -> None:
         authentication_keys=authentication_keys,
         max_retries=args.max_retries,
         max_wait_time=args.max_wait_time,
-        node_config=parse_config_args(args.node_config),
-        flwr_dir=get_flwr_dir(args.flwr_dir),
+        node_config=parse_config_args([args.node_config]),
+        flwr_path=get_flwr_dir(args.flwr_dir),
     )
 
     # Graceful shutdown
@@ -107,7 +107,7 @@ def run_client_app() -> None:
 
     _start_client_internal(
         server_address=args.superlink,
-        node_config=parse_config_args(args.node_config),
+        node_config=parse_config_args([args.node_config]),
         load_client_app_fn=load_fn,
         transport=args.transport,
         root_certificates=root_certificates,
@@ -248,7 +248,7 @@ def _get_load_client_app_fn(
             dir_path = Path(project_dir).absolute()
 
             # Set app reference
-            client_app_ref = config["tool"]["flwr"]["components"]["clientapp"]
+            client_app_ref = config["tool"]["flwr"]["app"]["components"]["clientapp"]
 
         # Set sys.path
         nonlocal inserted_path

flwr/common/config.py

@@ -86,6 +86,18 @@ def _fuse_dicts(
     return fused_dict
 
 
+def get_fused_config_from_dir(
+    project_dir: Path, override_config: Dict[str, str]
+) -> Dict[str, str]:
+    """Merge the overrides from a given dict with the config from a Flower App."""
+    default_config = get_project_config(project_dir)["tool"]["flwr"]["app"].get(
+        "config", {}
+    )
+    flat_default_config = flatten_dict(default_config)
+
+    return _fuse_dicts(flat_default_config, override_config)
+
+
 def get_fused_config(run: Run, flwr_dir: Optional[Path]) -> Dict[str, str]:
     """Merge the overrides from a `Run` with the config from a FAB.
 
@@ -97,10 +109,7 @@ def get_fused_config(run: Run, flwr_dir: Optional[Path]) -> Dict[str, str]:
 
     project_dir = get_project_dir(run.fab_id, run.fab_version, flwr_dir)
 
-    default_config = get_project_config(project_dir)["tool"]["flwr"].get("config", {})
-    flat_default_config = flatten_dict(default_config)
-
-    return _fuse_dicts(flat_default_config, run.override_config)
+    return get_fused_config_from_dir(project_dir, run.override_config)
 
 
 def flatten_dict(raw_dict: Dict[str, Any], parent_key: str = "") -> Dict[str, str]:
@@ -121,7 +130,7 @@ def flatten_dict(raw_dict: Dict[str, Any], parent_key: str = "") -> Dict[str, st
 
 
 def parse_config_args(
-    config: Optional[str],
+    config: Optional[List[str]],
     separator: str = ",",
 ) -> Dict[str, str]:
     """Parse separator separated list of key-value pairs separated by '='."""
@@ -130,17 +139,19 @@ def parse_config_args(
     if config is None:
         return overrides
 
-    overrides_list = config.split(separator)
-    if (
-        len(overrides_list) == 1
-        and "=" not in overrides_list
-        and overrides_list[0].endswith(".toml")
-    ):
-        with Path(overrides_list[0]).open("rb") as config_file:
-            overrides = flatten_dict(tomli.load(config_file))
-    else:
-        for kv_pair in overrides_list:
-            key, value = kv_pair.split("=")
-            overrides[key] = value
+    for config_line in config:
+        if config_line:
+            overrides_list = config_line.split(separator)
+            if (
+                len(overrides_list) == 1
+                and "=" not in overrides_list
+                and overrides_list[0].endswith(".toml")
+            ):
+                with Path(overrides_list[0]).open("rb") as config_file:
+                    overrides = flatten_dict(tomli.load(config_file))
+            else:
+                for kv_pair in overrides_list:
+                    key, value = kv_pair.split("=")
+                    overrides[key] = value
 
     return overrides

flwr/server/run_serverapp.py

@@ -186,7 +186,7 @@ def run_server_app() -> None:  # pylint: disable=too-many-branches
         run_ = driver.run
         server_app_dir = str(get_project_dir(run_.fab_id, run_.fab_version, flwr_dir))
         config = get_project_config(server_app_dir)
-        server_app_attr = config["tool"]["flwr"]["components"]["serverapp"]
+        server_app_attr = config["tool"]["flwr"]["app"]["components"]["serverapp"]
         server_app_run_config = get_fused_config(run_, flwr_dir)
     else:
         # User provided `server-app`, but not `--run-id`

flwr/server/superlink/fleet/vce/vce_api.py

@@ -61,7 +61,9 @@ def _register_nodes(
 
 
 def _register_node_states(
-    nodes_mapping: NodeToPartitionMapping, run: Run
+    nodes_mapping: NodeToPartitionMapping,
+    run: Run,
+    app_dir: Optional[str] = None,
 ) -> Dict[int, NodeState]:
     """Create NodeState objects and pre-register the context for the run."""
     node_states: Dict[int, NodeState] = {}
@@ -76,7 +78,9 @@ def _register_node_states(
         )
 
         # Pre-register Context objects
-        node_states[node_id].register_context(run_id=run.run_id, run=run)
+        node_states[node_id].register_context(
+            run_id=run.run_id, run=run, app_dir=app_dir
+        )
 
     return node_states
 
@@ -256,6 +260,7 @@ def start_vce(
     backend_name: str,
     backend_config_json_stream: str,
     app_dir: str,
+    is_app: bool,
     f_stop: threading.Event,
     run: Run,
     flwr_dir: Optional[str] = None,
@@ -309,7 +314,9 @@ def start_vce(
         )
 
     # Construct mapping of NodeStates
-    node_states = _register_node_states(nodes_mapping=nodes_mapping, run=run)
+    node_states = _register_node_states(
+        nodes_mapping=nodes_mapping, run=run, app_dir=app_dir if is_app else None
+    )
 
     # Load backend config
     log(DEBUG, "Supported backends: %s", list(supported_backends.keys()))

flwr/simulation/run_simulation.py

@@ -18,14 +18,19 @@ import argparse
 import asyncio
 import json
 import logging
+import sys
 import threading
 import traceback
+from argparse import Namespace
 from logging import DEBUG, ERROR, INFO, WARNING
+from pathlib import Path
 from time import sleep
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
+from flwr.cli.config_utils import load_and_validate
 from flwr.client import ClientApp
 from flwr.common import EventType, event, log
+from flwr.common.config import get_fused_config_from_dir, parse_config_args
 from flwr.common.constant import RUN_ID_NUM_BYTES
 from flwr.common.logger import set_logger_propagation, update_console_handler
 from flwr.common.typing import Run
@@ -41,28 +46,129 @@ from flwr.simulation.ray_transport.utils import (
 )
 
 
+def _check_args_do_not_interfere(args: Namespace) -> bool:
+    """Ensure decoupling of flags for different ways to start the simulation."""
+    mode_one_args = ["app", "run_config"]
+    mode_two_args = ["client_app", "server_app"]
+
+    def _resolve_message(conflict_keys: List[str]) -> str:
+        return ",".join([f"`--{key}`".replace("_", "-") for key in conflict_keys])
+
+    # When passing `--app`, `--app-dir` is ignored
+    if args.app and args.app_dir:
+        log(ERROR, "Either `--app` or `--app-dir` can be set, but not both.")
+        return False
+
+    if any(getattr(args, key) for key in mode_one_args):
+        if any(getattr(args, key) for key in mode_two_args):
+            log(
+                ERROR,
+                "Passing any of {%s} alongside with any of {%s}",
+                _resolve_message(mode_one_args),
+                _resolve_message(mode_two_args),
+            )
+            return False
+
+        if not args.app:
+            log(ERROR, "You need to pass --app")
+            return False
+
+        return True
+
+    # Ensure all args are set (required for the non-FAB mode of execution)
+    if not all(getattr(args, key) for key in mode_two_args):
+        log(
+            ERROR,
+            "Passing all of %s keys are required.",
+            _resolve_message(mode_two_args),
+        )
+        return False
+
+    return True
+
+
 # Entry point from CLI
+# pylint: disable=too-many-locals
 def run_simulation_from_cli() -> None:
     """Run Simulation Engine from the CLI."""
     args = _parse_args_run_simulation().parse_args()
 
+    # We are supporting two modes for the CLI entrypoint:
+    # 1) Running an app dir containing a `pyproject.toml`
+    # 2) Running any ClientApp and SeverApp w/o pyproject.toml being present
+    # For 2), some CLI args are compulsory, but they are not required for 1)
+    # We first do these checks
+    args_check_pass = _check_args_do_not_interfere(args)
+    if not args_check_pass:
+        sys.exit("Simulation Engine cannot start.")
+
+    run_id = (
+        generate_rand_int_from_bytes(RUN_ID_NUM_BYTES)
+        if args.run_id is None
+        else args.run_id
+    )
+    if args.app:
+        # Mode 1
+        app_path = Path(args.app)
+        if not app_path.is_dir():
+            log(ERROR, "--app is not a directory")
+            sys.exit("Simulation Engine cannot start.")
+
+        # Load pyproject.toml
+        config, errors, warnings = load_and_validate(
+            app_path / "pyproject.toml", check_module=False
+        )
+        if errors:
+            raise ValueError(errors)
+
+        if warnings:
+            log(WARNING, warnings)
+
+        if config is None:
+            raise ValueError("Config extracted from FAB's pyproject.toml is not valid")
+
+        # Get ClientApp and SeverApp components
+        app_components = config["tool"]["flwr"]["app"]["components"]
+        client_app_attr = app_components["clientapp"]
+        server_app_attr = app_components["serverapp"]
+
+        override_config = parse_config_args([args.run_config])
+        fused_config = get_fused_config_from_dir(app_path, override_config)
+        app_dir = args.app
+        is_app = True
+
+    else:
+        # Mode 2
+        client_app_attr = args.client_app
+        server_app_attr = args.server_app
+        override_config = {}
+        fused_config = None
+        app_dir = args.app_dir
+        is_app = False
+
+    # Create run
+    run = Run(
+        run_id=run_id,
+        fab_id="",
+        fab_version="",
+        override_config=override_config,
+    )
+
     # Load JSON config
     backend_config_dict = json.loads(args.backend_config)
 
     _run_simulation(
-        server_app_attr=args.server_app,
-        client_app_attr=args.client_app,
+        server_app_attr=server_app_attr,
+        client_app_attr=client_app_attr,
         num_supernodes=args.num_supernodes,
         backend_name=args.backend,
         backend_config=backend_config_dict,
-        app_dir=args.app_dir,
-        run=(
-            Run(run_id=args.run_id, fab_id="", fab_version="", override_config={})
-            if args.run_id
-            else None
-        ),
+        app_dir=app_dir,
+        run=run,
         enable_tf_gpu_growth=args.enable_tf_gpu_growth,
         verbose_logging=args.verbose,
+        server_app_run_config=fused_config,
+        is_app=is_app,
     )
 
 
@@ -205,6 +311,7 @@ def _main_loop(
     backend_name: str,
     backend_config_stream: str,
     app_dir: str,
+    is_app: bool,
     enable_tf_gpu_growth: bool,
     run: Run,
     flwr_dir: Optional[str] = None,
@@ -212,6 +319,7 @@ def _main_loop(
     client_app_attr: Optional[str] = None,
     server_app: Optional[ServerApp] = None,
     server_app_attr: Optional[str] = None,
+    server_app_run_config: Optional[Dict[str, str]] = None,
 ) -> None:
     """Launch SuperLink with Simulation Engine, then ServerApp on a separate thread."""
     # Initialize StateFactory
@@ -225,7 +333,9 @@ def _main_loop(
         # Register run
         log(DEBUG, "Pre-registering run with id %s", run.run_id)
         state_factory.state().run_ids[run.run_id] = run  # type: ignore
-        server_app_run_config: Dict[str, str] = {}
+
+        if server_app_run_config is None:
+            server_app_run_config = {}
 
         # Initialize Driver
         driver = InMemoryDriver(run_id=run.run_id, state_factory=state_factory)
@@ -251,6 +361,7 @@ def _main_loop(
             backend_name=backend_name,
             backend_config_json_stream=backend_config_stream,
             app_dir=app_dir,
+            is_app=is_app,
             state_factory=state_factory,
             f_stop=f_stop,
             run=run,
@@ -284,11 +395,13 @@ def _run_simulation(
     backend_config: Optional[BackendConfig] = None,
     client_app_attr: Optional[str] = None,
     server_app_attr: Optional[str] = None,
+    server_app_run_config: Optional[Dict[str, str]] = None,
     app_dir: str = "",
     flwr_dir: Optional[str] = None,
     run: Optional[Run] = None,
     enable_tf_gpu_growth: bool = False,
     verbose_logging: bool = False,
+    is_app: bool = False,
 ) -> None:
     r"""Launch the Simulation Engine.
 
@@ -317,14 +430,18 @@ def _run_simulation(
         parameters. Values supported in <value> are those included by
         `flwr.common.typing.ConfigsRecordValues`.
 
-    client_app_attr : str
+    client_app_attr : Optional[str]
         A path to a `ClientApp` module to be loaded: For example: `client:app` or
         `project.package.module:wrapper.app`."
 
-    server_app_attr : str
+    server_app_attr : Optional[str]
         A path to a `ServerApp` module to be loaded: For example: `server:app` or
         `project.package.module:wrapper.app`."
 
+    server_app_run_config : Optional[Dict[str, str]]
+        Config dictionary that parameterizes the run config. It will be made accesible
+        to the ServerApp.
+
     app_dir : str
         Add specified directory to the PYTHONPATH and load `ClientApp` from there.
         (Default: current working directory.)
@@ -346,6 +463,11 @@ def _run_simulation(
     verbose_logging : bool (default: False)
         When disabled, only INFO, WARNING and ERROR log messages will be shown. If
         enabled, DEBUG-level logs will be displayed.
+
+    is_app : bool (default: False)
+        A flag that indicates whether the simulation is running an app or not. This is
+        needed in order to attempt loading an app's pyproject.toml when nodes register
+        a context object.
     """
     if backend_config is None:
         backend_config = {}
@@ -381,6 +503,7 @@ def _run_simulation(
         backend_name,
         backend_config_stream,
         app_dir,
+        is_app,
         enable_tf_gpu_growth,
         run,
         flwr_dir,
@@ -388,6 +511,7 @@ def _run_simulation(
         client_app_attr,
         server_app,
         server_app_attr,
+        server_app_run_config,
     )
     # Detect if there is an Asyncio event loop already running.
     # If yes, disable logger propagation. In environmnets
@@ -419,12 +543,10 @@ def _parse_args_run_simulation() -> argparse.ArgumentParser:
     )
     parser.add_argument(
         "--server-app",
-        required=True,
         help="For example: `server:app` or `project.package.module:wrapper.app`",
     )
     parser.add_argument(
         "--client-app",
-        required=True,
         help="For example: `client:app` or `project.package.module:wrapper.app`",
     )
     parser.add_argument(
@@ -433,6 +555,18 @@ def _parse_args_run_simulation() -> argparse.ArgumentParser:
         required=True,
         help="Number of simulated SuperNodes.",
     )
+    parser.add_argument(
+        "--app",
+        type=str,
+        default=None,
+        help="Path to a directory containing a FAB-like structure with a "
+        "pyproject.toml.",
+    )
+    parser.add_argument(
+        "--run-config",
+        default=None,
+        help="Override configuration key-value pairs.",
+    )
     parser.add_argument(
         "--backend",
         default="ray",

flwr/superexec/app.py

@@ -56,7 +56,7 @@ def run_superexec() -> None:
         address=address,
         executor=_load_executor(args),
         certificates=certificates,
-        config=parse_config_args(args.executor_config),
+        config=parse_config_args([args.executor_config]),
     )
 
     grpc_servers = [superexec_server]

flwr/superexec/deployment.py

@@ -15,7 +15,6 @@
 """Deployment engine executor."""
 
 import subprocess
-import sys
 from logging import ERROR, INFO
 from pathlib import Path
 from typing import Dict, Optional
@@ -131,14 +130,7 @@ class DeploymentEngine(Executor):
         try:
             # Install FAB to flwr dir
             fab_version, fab_id = get_fab_metadata(fab_file)
-            fab_path = install_from_fab(fab_file, None, True)
-
-            # Install FAB Python package
-            subprocess.check_call(
-                [sys.executable, "-m", "pip", "install", "--no-deps", str(fab_path)],
-                stdout=subprocess.DEVNULL,
-                stderr=subprocess.DEVNULL,
-            )
+            install_from_fab(fab_file, None, True)
 
             # Call SuperLink to create run
             run_id: int = self._create_run(fab_id, fab_version, override_config)

flwr/superexec/simulation.py

@@ -82,11 +82,6 @@ class SimulationEngine(Executor):
     ) -> Optional[RunTracker]:
         """Start run using the Flower Simulation Engine."""
         try:
-            if override_config:
-                raise ValueError(
-                    "Overriding the run config is not yet supported with the "
-                    "simulation executor.",
-                )
 
             # Install FAB to flwr dir
             fab_path = install_from_fab(fab_file, None, True)
@@ -111,11 +106,6 @@ class SimulationEngine(Executor):
                     "Config extracted from FAB's pyproject.toml is not valid"
                 )
 
-            # Get ClientApp and SeverApp components
-            flower_components = config["tool"]["flwr"]["components"]
-            clientapp = flower_components["clientapp"]
-            serverapp = flower_components["serverapp"]
-
             # In Simulation there is no SuperLink, still we create a run_id
             run_id = generate_rand_int_from_bytes(RUN_ID_NUM_BYTES)
             log(INFO, "Created run %s", str(run_id))
@@ -123,21 +113,21 @@ class SimulationEngine(Executor):
             # Prepare commnand
             command = [
                 "flower-simulation",
-                "--client-app",
-                f"{clientapp}",
-                "--server-app",
-                f"{serverapp}",
+                "--app",
+                f"{str(fab_path)}",
                 "--num-supernodes",
                 f"{self.num_supernodes}",
                 "--run-id",
                 str(run_id),
             ]
 
+            if override_config:
+                command.extend(["--run-config", f"{override_config}"])
+
             # Start Simulation
-            proc = subprocess.Popen(  # pylint: disable=consider-using-with
+            proc = subprocess.run(  # pylint: disable=consider-using-with
                 command,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
+                check=True,
                 text=True,
             )
 
@@ -145,7 +135,7 @@ class SimulationEngine(Executor):
 
             return RunTracker(
                 run_id=run_id,
-                proc=proc,
+                proc=proc,  # type:ignore
             )
 
         # pylint: disable-next=broad-except

{flwr_nightly-1.10.0.dev20240715.dist-info → flwr_nightly-1.10.0.dev20240717.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.10.0.dev20240715
+Version: 1.10.0.dev20240717
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0
@@ -43,6 +43,7 @@ Requires-Dist: ray (==2.10.0) ; (python_version >= "3.8" and python_version < "3
 Requires-Dist: requests (>=2.31.0,<3.0.0) ; extra == "rest"
 Requires-Dist: starlette (>=0.31.0,<0.32.0) ; extra == "rest"
 Requires-Dist: tomli (>=2.0.1,<3.0.0)
+Requires-Dist: tomli-w (>=1.0.0,<2.0.0)
 Requires-Dist: typer[all] (>=0.9.0,<0.10.0)
 Requires-Dist: uvicorn[standard] (>=0.23.0,<0.24.0) ; extra == "rest"
 Project-URL: Documentation, https://flower.ai