flwr-nightly 1.13.0.dev20241106__py3-none-any.whl → 1.13.0.dev20241117__py3-none-any.whl

flwr/simulation/legacy_app.py

@@ -0,0 +1,402 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower simulation app."""
+
+
+import asyncio
+import logging
+import sys
+import threading
+import traceback
+import warnings
+from logging import ERROR, INFO
+from typing import Any, Optional, Union
+
+import ray
+from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
+
+from flwr.client import ClientFnExt
+from flwr.common import EventType, event
+from flwr.common.constant import NODE_ID_NUM_BYTES
+from flwr.common.logger import (
+    log,
+    set_logger_propagation,
+    warn_deprecated_feature,
+    warn_unsupported_feature,
+)
+from flwr.server.client_manager import ClientManager
+from flwr.server.history import History
+from flwr.server.server import Server, init_defaults, run_fl
+from flwr.server.server_config import ServerConfig
+from flwr.server.strategy import Strategy
+from flwr.server.superlink.linkstate.utils import generate_rand_int_from_bytes
+from flwr.simulation.ray_transport.ray_actor import (
+    ClientAppActor,
+    VirtualClientEngineActor,
+    VirtualClientEngineActorPool,
+    pool_size_from_resources,
+)
+from flwr.simulation.ray_transport.ray_client_proxy import RayActorClientProxy
+
+INVALID_ARGUMENTS_START_SIMULATION = """
+INVALID ARGUMENTS ERROR
+
+Invalid Arguments in method:
+
+`start_simulation(
+    *,
+    client_fn: ClientFn,
+    num_clients: int,
+    clients_ids: Optional[List[str]] = None,
+    client_resources: Optional[Dict[str, float]] = None,
+    server: Optional[Server] = None,
+    config: ServerConfig = None,
+    strategy: Optional[Strategy] = None,
+    client_manager: Optional[ClientManager] = None,
+    ray_init_args: Optional[Dict[str, Any]] = None,
+) -> None:`
+
+REASON:
+    Method requires:
+        - Either `num_clients`[int] or `clients_ids`[List[str]]
+        to be set exclusively.
+        OR
+        - `len(clients_ids)` == `num_clients`
+
+"""
+
+NodeToPartitionMapping = dict[int, int]
+
+
+def _create_node_id_to_partition_mapping(
+    num_clients: int,
+) -> NodeToPartitionMapping:
+    """Generate a node_id:partition_id mapping."""
+    nodes_mapping: NodeToPartitionMapping = {}  # {node-id; partition-id}
+    for i in range(num_clients):
+        while True:
+            node_id = generate_rand_int_from_bytes(NODE_ID_NUM_BYTES)
+            if node_id not in nodes_mapping:
+                break
+        nodes_mapping[node_id] = i
+    return nodes_mapping
+
+
+# pylint: disable=too-many-arguments,too-many-statements,too-many-branches
+def start_simulation(
+    *,
+    client_fn: ClientFnExt,
+    num_clients: int,
+    clients_ids: Optional[list[str]] = None,  # UNSUPPORTED, WILL BE REMOVED
+    client_resources: Optional[dict[str, float]] = None,
+    server: Optional[Server] = None,
+    config: Optional[ServerConfig] = None,
+    strategy: Optional[Strategy] = None,
+    client_manager: Optional[ClientManager] = None,
+    ray_init_args: Optional[dict[str, Any]] = None,
+    keep_initialised: Optional[bool] = False,
+    actor_type: type[VirtualClientEngineActor] = ClientAppActor,
+    actor_kwargs: Optional[dict[str, Any]] = None,
+    actor_scheduling: Union[str, NodeAffinitySchedulingStrategy] = "DEFAULT",
+) -> History:
+    """Start a Ray-based Flower simulation server.
+
+    Warning
+    -------
+    This function is deprecated since 1.13.0. Use :code: `flwr run` to start a Flower
+    simulation.
+
+    Parameters
+    ----------
+    client_fn : ClientFnExt
+        A function creating `Client` instances. The function must have the signature
+        `client_fn(context: Context). It should return
+        a single client instance of type `Client`. Note that the created client
+        instances are ephemeral and will often be destroyed after a single method
+        invocation. Since client instances are not long-lived, they should not attempt
+        to carry state over method invocations. Any state required by the instance
+        (model, dataset, hyperparameters, ...) should be (re-)created in either the
+        call to `client_fn` or the call to any of the client methods (e.g., load
+        evaluation data in the `evaluate` method itself).
+    num_clients : int
+        The total number of clients in this simulation.
+    clients_ids : Optional[List[str]]
+        UNSUPPORTED, WILL BE REMOVED. USE `num_clients` INSTEAD.
+        List `client_id`s for each client. This is only required if
+        `num_clients` is not set. Setting both `num_clients` and `clients_ids`
+        with `len(clients_ids)` not equal to `num_clients` generates an error.
+        Using this argument will raise an error.
+    client_resources : Optional[Dict[str, float]] (default: `{"num_cpus": 1, "num_gpus": 0.0}`)
+        CPU and GPU resources for a single client. Supported keys
+        are `num_cpus` and `num_gpus`. To understand the GPU utilization caused by
+        `num_gpus`, as well as using custom resources, please consult the Ray
+        documentation.
+    server : Optional[flwr.server.Server] (default: None).
+        An implementation of the abstract base class `flwr.server.Server`. If no
+        instance is provided, then `start_server` will create one.
+    config: ServerConfig (default: None).
+        Currently supported values are `num_rounds` (int, default: 1) and
+        `round_timeout` in seconds (float, default: None).
+    strategy : Optional[flwr.server.Strategy] (default: None)
+        An implementation of the abstract base class `flwr.server.Strategy`. If
+        no strategy is provided, then `start_server` will use
+        `flwr.server.strategy.FedAvg`.
+    client_manager : Optional[flwr.server.ClientManager] (default: None)
+        An implementation of the abstract base class `flwr.server.ClientManager`.
+        If no implementation is provided, then `start_simulation` will use
+        `flwr.server.client_manager.SimpleClientManager`.
+    ray_init_args : Optional[Dict[str, Any]] (default: None)
+        Optional dictionary containing arguments for the call to `ray.init`.
+        If ray_init_args is None (the default), Ray will be initialized with
+        the following default args:
+
+        { "ignore_reinit_error": True, "include_dashboard": False }
+
+        An empty dictionary can be used (ray_init_args={}) to prevent any
+        arguments from being passed to ray.init.
+    keep_initialised: Optional[bool] (default: False)
+        Set to True to prevent `ray.shutdown()` in case `ray.is_initialized()=True`.
+
+    actor_type: VirtualClientEngineActor (default: ClientAppActor)
+        Optionally specify the type of actor to use. The actor object, which
+        persists throughout the simulation, will be the process in charge of
+        executing a ClientApp wrapping input argument `client_fn`.
+
+    actor_kwargs: Optional[Dict[str, Any]] (default: None)
+        If you want to create your own Actor classes, you might need to pass
+        some input argument. You can use this dictionary for such purpose.
+
+    actor_scheduling: Optional[Union[str, NodeAffinitySchedulingStrategy]]
+        (default: "DEFAULT")
+        Optional string ("DEFAULT" or "SPREAD") for the VCE to choose in which
+        node the actor is placed. If you are an advanced user needed more control
+        you can use lower-level scheduling strategies to pin actors to specific
+        compute nodes (e.g. via NodeAffinitySchedulingStrategy). Please note this
+        is an advanced feature. For all details, please refer to the Ray documentation:
+        https://docs.ray.io/en/latest/ray-core/scheduling/index.html
+
+    Returns
+    -------
+    hist : flwr.server.history.History
+        Object containing metrics from training.
+    """  # noqa: E501
+    # pylint: disable-msg=too-many-locals
+    msg = (
+        "flwr.simulation.start_simulation() is deprecated."
+        "\n\tInstead, use the `flwr run` CLI command to start a local simulation "
+        "in your Flower app, as shown for example below:"
+        "\n\n\t\t$ flwr new  # Create a new Flower app from a template"
+        "\n\n\t\t$ flwr run  # Run the Flower app in Simulation Mode"
+        "\n\n\tUsing `start_simulation()` is deprecated."
+    )
+    warn_deprecated_feature(name=msg)
+
+    event(
+        EventType.START_SIMULATION_ENTER,
+        {"num_clients": len(clients_ids) if clients_ids is not None else num_clients},
+    )
+
+    if clients_ids is not None:
+        warn_unsupported_feature(
+            "Passing `clients_ids` to `start_simulation` is deprecated and not longer "
+            "used by `start_simulation`. Use `num_clients` exclusively instead."
+        )
+        log(ERROR, "`clients_ids` argument used.")
+        sys.exit()
+
+    # Set logger propagation
+    loop: Optional[asyncio.AbstractEventLoop] = None
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        loop = None
+    finally:
+        if loop and loop.is_running():
+            # Set logger propagation to False to prevent duplicated log output in Colab.
+            logger = logging.getLogger("flwr")
+            _ = set_logger_propagation(logger, False)
+
+    # Initialize server and server config
+    initialized_server, initialized_config = init_defaults(
+        server=server,
+        config=config,
+        strategy=strategy,
+        client_manager=client_manager,
+    )
+
+    log(
+        INFO,
+        "Starting Flower simulation, config: %s",
+        initialized_config,
+    )
+
+    # Create node-id to partition-id mapping
+    nodes_mapping = _create_node_id_to_partition_mapping(num_clients)
+
+    # Default arguments for Ray initialization
+    if not ray_init_args:
+        ray_init_args = {
+            "ignore_reinit_error": True,
+            "include_dashboard": False,
+        }
+
+    # Shut down Ray if it has already been initialized (unless asked not to)
+    if ray.is_initialized() and not keep_initialised:
+        ray.shutdown()
+
+    # Initialize Ray
+    ray.init(**ray_init_args)
+    cluster_resources = ray.cluster_resources()
+    log(
+        INFO,
+        "Flower VCE: Ray initialized with resources: %s",
+        cluster_resources,
+    )
+
+    log(
+        INFO,
+        "Optimize your simulation with Flower VCE: "
+        "https://flower.ai/docs/framework/how-to-run-simulations.html",
+    )
+
+    # Log the resources that a single client will be able to use
+    if client_resources is None:
+        log(
+            INFO,
+            "No `client_resources` specified. Using minimal resources for clients.",
+        )
+        client_resources = {"num_cpus": 1, "num_gpus": 0.0}
+
+    # Each client needs at the very least one CPU
+    if "num_cpus" not in client_resources:
+        warnings.warn(
+            "No `num_cpus` specified in `client_resources`. "
+            "Using `num_cpus=1` for each client.",
+            stacklevel=2,
+        )
+        client_resources["num_cpus"] = 1
+
+    log(
+        INFO,
+        "Flower VCE: Resources for each Virtual Client: %s",
+        client_resources,
+    )
+
+    actor_args = {} if actor_kwargs is None else actor_kwargs
+
+    # An actor factory. This is called N times to add N actors
+    # to the pool. If at some point the pool can accommodate more actors
+    # this will be called again.
+    def create_actor_fn() -> type[VirtualClientEngineActor]:
+        return actor_type.options(  # type: ignore
+            **client_resources,
+            scheduling_strategy=actor_scheduling,
+        ).remote(**actor_args)
+
+    # Instantiate ActorPool
+    pool = VirtualClientEngineActorPool(
+        create_actor_fn=create_actor_fn,
+        client_resources=client_resources,
+    )
+
+    f_stop = threading.Event()
+
+    # Periodically, check if the cluster has grown (i.e. a new
+    # node has been added). If this happens, we likely want to grow
+    # the actor pool by adding more Actors to it.
+    def update_resources(f_stop: threading.Event) -> None:
+        """Periodically check if more actors can be added to the pool.
+
+        If so, extend the pool.
+        """
+        if not f_stop.is_set():
+            num_max_actors = pool_size_from_resources(client_resources)
+            if num_max_actors > pool.num_actors:
+                num_new = num_max_actors - pool.num_actors
+                log(
+                    INFO, "The cluster expanded. Adding %s actors to the pool.", num_new
+                )
+                pool.add_actors_to_pool(num_actors=num_new)
+
+            threading.Timer(10, update_resources, [f_stop]).start()
+
+    update_resources(f_stop)
+
+    log(
+        INFO,
+        "Flower VCE: Creating %s with %s actors",
+        pool.__class__.__name__,
+        pool.num_actors,
+    )
+
+    # Register one RayClientProxy object for each client with the ClientManager
+    for node_id, partition_id in nodes_mapping.items():
+        client_proxy = RayActorClientProxy(
+            client_fn=client_fn,
+            node_id=node_id,
+            partition_id=partition_id,
+            num_partitions=num_clients,
+            actor_pool=pool,
+        )
+        initialized_server.client_manager().register(client=client_proxy)
+
+    hist = History()
+    # pylint: disable=broad-except
+    try:
+        # Start training
+        hist = run_fl(
+            server=initialized_server,
+            config=initialized_config,
+        )
+    except Exception as ex:
+        log(ERROR, ex)
+        log(ERROR, traceback.format_exc())
+        log(
+            ERROR,
+            "Your simulation crashed :(. This could be because of several reasons. "
+            "The most common are: "
+            "\n\t > Sometimes, issues in the simulation code itself can cause crashes. "
+            "It's always a good idea to double-check your code for any potential bugs "
+            "or inconsistencies that might be contributing to the problem. "
+            "For example: "
+            "\n\t\t - You might be using a class attribute in your clients that "
+            "hasn't been defined."
+            "\n\t\t - There could be an incorrect method call to a 3rd party library "
+            "(e.g., PyTorch)."
+            "\n\t\t - The return types of methods in your clients/strategies might be "
+            "incorrect."
+            "\n\t > Your system couldn't fit a single VirtualClient: try lowering "
+            "`client_resources`."
+            "\n\t > All the actors in your pool crashed. This could be because: "
+            "\n\t\t - You clients hit an out-of-memory (OOM) error and actors couldn't "
+            "recover from it. Try launching your simulation with more generous "
+            "`client_resources` setting (i.e. it seems %s is "
+            "not enough for your run). Use fewer concurrent actors. "
+            "\n\t\t - You were running a multi-node simulation and all worker nodes "
+            "disconnected. The head node might still be alive but cannot accommodate "
+            "any actor with resources: %s."
+            "\nTake a look at the Flower simulation examples for guidance "
+            "<https://flower.ai/docs/framework/how-to-run-simulations.html>.",
+            client_resources,
+            client_resources,
+        )
+        raise RuntimeError("Simulation crashed.") from ex
+
+    finally:
+        # Stop time monitoring resources in cluster
+        f_stop.set()
+        event(EventType.START_SIMULATION_LEAVE)
+
+    return hist

flwr/simulation/run_simulation.py

@@ -123,13 +123,8 @@ def run_simulation_from_cli() -> None:
     fused_config = get_fused_config_from_dir(app_path, override_config)
 
     # Create run
-    run = Run(
-        run_id=run_id,
-        fab_id="",
-        fab_version="",
-        fab_hash="",
-        override_config=override_config,
-    )
+    run = Run.create_empty(run_id)
+    run.override_config = override_config
 
     _run_simulation(
         server_app_attr=server_app_attr,
@@ -333,21 +328,17 @@ def _main_loop(
     try:
         # Register run
         log(DEBUG, "Pre-registering run with id %s", run.run_id)
-        init_status = RunStatus(Status.RUNNING, "", "")
-        state_factory.state().run_ids[run.run_id] = RunRecord(  # type: ignore
-            run=run,
-            status=init_status,
-            starting_at=now().isoformat(),
-            running_at=now().isoformat(),
-            finished_at="",
-        )
+        run.status = RunStatus(Status.RUNNING, "", "")
+        run.starting_at = now().isoformat()
+        run.running_at = run.starting_at
+        state_factory.state().run_ids[run.run_id] = RunRecord(run=run)  # type: ignore
 
         if server_app_run_config is None:
             server_app_run_config = {}
 
         # Initialize Driver
         driver = InMemoryDriver(state_factory=state_factory)
-        driver.init_run(run_id=run.run_id)
+        driver.set_run(run_id=run.run_id)
 
         # Get and run ServerApp thread
         serverapp_th = run_serverapp_th(
@@ -457,9 +448,7 @@ def _run_simulation(
     # If no `Run` object is set, create one
     if run is None:
         run_id = generate_rand_int_from_bytes(RUN_ID_NUM_BYTES)
-        run = Run(
-            run_id=run_id, fab_id="", fab_version="", fab_hash="", override_config={}
-        )
+        run = Run.create_empty(run_id=run_id)
 
     args = (
         num_supernodes,

flwr/simulation/simulationio_connection.py

@@ -20,7 +20,7 @@ from typing import Optional, cast
 
 import grpc
 
-from flwr.common.constant import SIMULATIONIO_API_DEFAULT_ADDRESS
+from flwr.common.constant import SIMULATIONIO_API_DEFAULT_CLIENT_ADDRESS
 from flwr.common.grpc import create_channel
 from flwr.common.logger import log
 from flwr.proto.simulationio_pb2_grpc import SimulationIoStub  # pylint: disable=E0611
@@ -41,7 +41,7 @@ class SimulationIoConnection:
 
     def __init__(  # pylint: disable=too-many-arguments
         self,
-        simulationio_service_address: str = SIMULATIONIO_API_DEFAULT_ADDRESS,
+        simulationio_service_address: str = SIMULATIONIO_API_DEFAULT_CLIENT_ADDRESS,
         root_certificates: Optional[bytes] = None,
     ) -> None:
         self._addr = simulationio_service_address

flwr/superexec/deployment.py

@@ -21,8 +21,13 @@ from typing import Optional
 
 from typing_extensions import override
 
+from flwr.cli.config_utils import get_fab_metadata
 from flwr.common import ConfigsRecord, Context, RecordSet
-from flwr.common.constant import SERVERAPPIO_API_DEFAULT_ADDRESS, Status, SubStatus
+from flwr.common.constant import (
+    SERVERAPPIO_API_DEFAULT_CLIENT_ADDRESS,
+    Status,
+    SubStatus,
+)
 from flwr.common.logger import log
 from flwr.common.typing import Fab, RunStatus, UserConfig
 from flwr.server.superlink.ffs import Ffs
@@ -37,7 +42,7 @@ class DeploymentEngine(Executor):
 
     Parameters
     ----------
-    superlink: str (default: "0.0.0.0:9091")
+    serverappio_api_address: str (default: "127.0.0.1:9091")
         Address of the SuperLink to connect to.
     root_certificates: Optional[str] (default: None)
         Specifies the path to the PEM-encoded root certificate file for
@@ -48,11 +53,11 @@ class DeploymentEngine(Executor):
 
     def __init__(
         self,
-        superlink: str = SERVERAPPIO_API_DEFAULT_ADDRESS,
+        serverappio_api_address: str = SERVERAPPIO_API_DEFAULT_CLIENT_ADDRESS,
         root_certificates: Optional[str] = None,
         flwr_dir: Optional[str] = None,
     ) -> None:
-        self.superlink = superlink
+        self.serverappio_api_address = serverappio_api_address
         if root_certificates is None:
             self.root_certificates = None
             self.root_certificates_bytes = None
@@ -109,7 +114,7 @@ class DeploymentEngine(Executor):
         if superlink_address := config.get("superlink"):
             if not isinstance(superlink_address, str):
                 raise ValueError("The `superlink` value should be of type `str`.")
-            self.superlink = superlink_address
+            self.serverappio_api_address = superlink_address
         if root_certificates := config.get("root-certificates"):
             if not isinstance(root_certificates, str):
                 raise ValueError(
@@ -132,9 +137,10 @@ class DeploymentEngine(Executor):
             raise RuntimeError(
                 f"FAB ({fab.hash_str}) hash from request doesn't match contents"
             )
+        fab_id, fab_version = get_fab_metadata(fab.content)
 
         run_id = self.linkstate.create_run(
-            None, None, fab_hash, override_config, ConfigsRecord()
+            fab_id, fab_version, fab_hash, override_config, ConfigsRecord()
         )
         return run_id
 
@@ -153,7 +159,7 @@ class DeploymentEngine(Executor):
         self,
         fab_file: bytes,
         override_config: UserConfig,
-        federation_config: UserConfig,
+        federation_options: ConfigsRecord,
     ) -> Optional[int]:
         """Start run using the Flower Deployment Engine."""
         run_id = None

flwr/superexec/exec_servicer.py

@@ -22,18 +22,25 @@ from typing import Any
 
 import grpc
 
+from flwr.common import now
 from flwr.common.constant import LOG_STREAM_INTERVAL, Status
 from flwr.common.logger import log
-from flwr.common.serde import user_config_from_proto
+from flwr.common.serde import (
+    configs_record_from_proto,
+    run_to_proto,
+    user_config_from_proto,
+)
 from flwr.proto import exec_pb2_grpc  # pylint: disable=E0611
 from flwr.proto.exec_pb2 import (  # pylint: disable=E0611
+    ListRunsRequest,
+    ListRunsResponse,
     StartRunRequest,
     StartRunResponse,
     StreamLogsRequest,
     StreamLogsResponse,
 )
 from flwr.server.superlink.ffs.ffs_factory import FfsFactory
-from flwr.server.superlink.linkstate import LinkStateFactory
+from flwr.server.superlink.linkstate import LinkState, LinkStateFactory
 
 from .executor import Executor
 
@@ -61,7 +68,7 @@ class ExecServicer(exec_pb2_grpc.ExecServicer):
         run_id = self.executor.start_run(
             request.fab.content,
             user_config_from_proto(request.override_config),
-            user_config_from_proto(request.federation_config),
+            configs_record_from_proto(request.federation_options),
         )
 
         if run_id is None:
@@ -105,3 +112,25 @@ class ExecServicer(exec_pb2_grpc.ExecServicer):
                 context.cancel()
 
             time.sleep(LOG_STREAM_INTERVAL)  # Sleep briefly to avoid busy waiting
+
+    def ListRuns(
+        self, request: ListRunsRequest, context: grpc.ServicerContext
+    ) -> ListRunsResponse:
+        """Handle `flwr ls` command."""
+        log(INFO, "ExecServicer.List")
+        state = self.linkstate_factory.state()
+
+        # Handle `flwr ls --runs`
+        if not request.HasField("run_id"):
+            return _create_list_runs_response(state.get_run_ids(), state)
+        # Handle `flwr ls --run-id <run_id>`
+        return _create_list_runs_response({request.run_id}, state)
+
+
+def _create_list_runs_response(run_ids: set[int], state: LinkState) -> ListRunsResponse:
+    """Create response for `flwr ls --runs` and `flwr ls --run-id <run_id>`."""
+    run_dict = {run_id: state.get_run(run_id) for run_id in run_ids}
+    return ListRunsResponse(
+        run_dict={run_id: run_to_proto(run) for run_id, run in run_dict.items() if run},
+        now=now().isoformat(),
+    )

flwr/superexec/executor.py

@@ -19,6 +19,7 @@ from dataclasses import dataclass, field
 from subprocess import Popen
 from typing import Optional
 
+from flwr.common import ConfigsRecord
 from flwr.common.typing import UserConfig
 from flwr.server.superlink.ffs.ffs_factory import FfsFactory
 from flwr.server.superlink.linkstate import LinkStateFactory
@@ -71,7 +72,7 @@ class Executor(ABC):
         self,
         fab_file: bytes,
         override_config: UserConfig,
-        federation_config: UserConfig,
+        federation_options: ConfigsRecord,
     ) -> Optional[int]:
         """Start a run using the given Flower FAB ID and version.
 
@@ -84,8 +85,8 @@ class Executor(ABC):
             The Flower App Bundle file bytes.
         override_config: UserConfig
             The config overrides dict sent by the user (using `flwr run`).
-        federation_config: UserConfig
-            The federation options dict sent by the user (using `flwr run`).
+        federation_options: ConfigsRecord
+            The federation options sent by the user (using `flwr run`).
 
         Returns
         -------