flwr-nightly 1.8.0.dev20240315__py3-none-any.whl → 1.15.0.dev20250115__py3-none-any.whl

flwr/simulation/legacy_app.py

@@ -0,0 +1,404 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower simulation app."""
+
+
+import asyncio
+import logging
+import sys
+import threading
+import traceback
+import warnings
+from logging import ERROR, INFO
+from typing import Any, Optional, Union
+
+import ray
+from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
+
+from flwr.client import ClientFnExt
+from flwr.common import EventType, event
+from flwr.common.constant import NODE_ID_NUM_BYTES, SUPERLINK_NODE_ID
+from flwr.common.logger import (
+    log,
+    set_logger_propagation,
+    warn_deprecated_feature,
+    warn_unsupported_feature,
+)
+from flwr.server.client_manager import ClientManager
+from flwr.server.history import History
+from flwr.server.server import Server, init_defaults, run_fl
+from flwr.server.server_config import ServerConfig
+from flwr.server.strategy import Strategy
+from flwr.server.superlink.linkstate.utils import generate_rand_int_from_bytes
+from flwr.simulation.ray_transport.ray_actor import (
+    ClientAppActor,
+    VirtualClientEngineActor,
+    VirtualClientEngineActorPool,
+    pool_size_from_resources,
+)
+from flwr.simulation.ray_transport.ray_client_proxy import RayActorClientProxy
+
+INVALID_ARGUMENTS_START_SIMULATION = """
+INVALID ARGUMENTS ERROR
+
+Invalid Arguments in method:
+
+`start_simulation(
+    *,
+    client_fn: ClientFn,
+    num_clients: int,
+    clients_ids: Optional[List[str]] = None,
+    client_resources: Optional[Dict[str, float]] = None,
+    server: Optional[Server] = None,
+    config: ServerConfig = None,
+    strategy: Optional[Strategy] = None,
+    client_manager: Optional[ClientManager] = None,
+    ray_init_args: Optional[Dict[str, Any]] = None,
+) -> None:`
+
+REASON:
+    Method requires:
+        - Either `num_clients`[int] or `clients_ids`[List[str]]
+        to be set exclusively.
+        OR
+        - `len(clients_ids)` == `num_clients`
+
+"""
+
+NodeToPartitionMapping = dict[int, int]
+
+
+def _create_node_id_to_partition_mapping(
+    num_clients: int,
+) -> NodeToPartitionMapping:
+    """Generate a node_id:partition_id mapping."""
+    nodes_mapping: NodeToPartitionMapping = {}  # {node-id; partition-id}
+    for i in range(num_clients):
+        while True:
+            node_id = generate_rand_int_from_bytes(
+                NODE_ID_NUM_BYTES, exclude=[SUPERLINK_NODE_ID, 0]
+            )
+            if node_id not in nodes_mapping:
+                break
+        nodes_mapping[node_id] = i
+    return nodes_mapping
+
+
+# pylint: disable=too-many-arguments,too-many-statements,too-many-branches
+def start_simulation(
+    *,
+    client_fn: ClientFnExt,
+    num_clients: int,
+    clients_ids: Optional[list[str]] = None,  # UNSUPPORTED, WILL BE REMOVED
+    client_resources: Optional[dict[str, float]] = None,
+    server: Optional[Server] = None,
+    config: Optional[ServerConfig] = None,
+    strategy: Optional[Strategy] = None,
+    client_manager: Optional[ClientManager] = None,
+    ray_init_args: Optional[dict[str, Any]] = None,
+    keep_initialised: Optional[bool] = False,
+    actor_type: type[VirtualClientEngineActor] = ClientAppActor,
+    actor_kwargs: Optional[dict[str, Any]] = None,
+    actor_scheduling: Union[str, NodeAffinitySchedulingStrategy] = "DEFAULT",
+) -> History:
+    """Start a Ray-based Flower simulation server.
+
+    Warning
+    -------
+    This function is deprecated since 1.13.0. Use :code: `flwr run` to start a Flower
+    simulation.
+
+    Parameters
+    ----------
+    client_fn : ClientFnExt
+        A function creating `Client` instances. The function must have the signature
+        `client_fn(context: Context). It should return
+        a single client instance of type `Client`. Note that the created client
+        instances are ephemeral and will often be destroyed after a single method
+        invocation. Since client instances are not long-lived, they should not attempt
+        to carry state over method invocations. Any state required by the instance
+        (model, dataset, hyperparameters, ...) should be (re-)created in either the
+        call to `client_fn` or the call to any of the client methods (e.g., load
+        evaluation data in the `evaluate` method itself).
+    num_clients : int
+        The total number of clients in this simulation.
+    clients_ids : Optional[List[str]]
+        UNSUPPORTED, WILL BE REMOVED. USE `num_clients` INSTEAD.
+        List `client_id`s for each client. This is only required if
+        `num_clients` is not set. Setting both `num_clients` and `clients_ids`
+        with `len(clients_ids)` not equal to `num_clients` generates an error.
+        Using this argument will raise an error.
+    client_resources : Optional[Dict[str, float]] (default: `{"num_cpus": 1, "num_gpus": 0.0}`)
+        CPU and GPU resources for a single client. Supported keys
+        are `num_cpus` and `num_gpus`. To understand the GPU utilization caused by
+        `num_gpus`, as well as using custom resources, please consult the Ray
+        documentation.
+    server : Optional[flwr.server.Server] (default: None).
+        An implementation of the abstract base class `flwr.server.Server`. If no
+        instance is provided, then `start_server` will create one.
+    config: ServerConfig (default: None).
+        Currently supported values are `num_rounds` (int, default: 1) and
+        `round_timeout` in seconds (float, default: None).
+    strategy : Optional[flwr.server.Strategy] (default: None)
+        An implementation of the abstract base class `flwr.server.Strategy`. If
+        no strategy is provided, then `start_server` will use
+        `flwr.server.strategy.FedAvg`.
+    client_manager : Optional[flwr.server.ClientManager] (default: None)
+        An implementation of the abstract base class `flwr.server.ClientManager`.
+        If no implementation is provided, then `start_simulation` will use
+        `flwr.server.client_manager.SimpleClientManager`.
+    ray_init_args : Optional[Dict[str, Any]] (default: None)
+        Optional dictionary containing arguments for the call to `ray.init`.
+        If ray_init_args is None (the default), Ray will be initialized with
+        the following default args:
+
+        { "ignore_reinit_error": True, "include_dashboard": False }
+
+        An empty dictionary can be used (ray_init_args={}) to prevent any
+        arguments from being passed to ray.init.
+    keep_initialised: Optional[bool] (default: False)
+        Set to True to prevent `ray.shutdown()` in case `ray.is_initialized()=True`.
+
+    actor_type: VirtualClientEngineActor (default: ClientAppActor)
+        Optionally specify the type of actor to use. The actor object, which
+        persists throughout the simulation, will be the process in charge of
+        executing a ClientApp wrapping input argument `client_fn`.
+
+    actor_kwargs: Optional[Dict[str, Any]] (default: None)
+        If you want to create your own Actor classes, you might need to pass
+        some input argument. You can use this dictionary for such purpose.
+
+    actor_scheduling: Optional[Union[str, NodeAffinitySchedulingStrategy]]
+        (default: "DEFAULT")
+        Optional string ("DEFAULT" or "SPREAD") for the VCE to choose in which
+        node the actor is placed. If you are an advanced user needed more control
+        you can use lower-level scheduling strategies to pin actors to specific
+        compute nodes (e.g. via NodeAffinitySchedulingStrategy). Please note this
+        is an advanced feature. For all details, please refer to the Ray documentation:
+        https://docs.ray.io/en/latest/ray-core/scheduling/index.html
+
+    Returns
+    -------
+    hist : flwr.server.history.History
+        Object containing metrics from training.
+    """  # noqa: E501
+    # pylint: disable-msg=too-many-locals
+    msg = (
+        "flwr.simulation.start_simulation() is deprecated."
+        "\n\tInstead, use the `flwr run` CLI command to start a local simulation "
+        "in your Flower app, as shown for example below:"
+        "\n\n\t\t$ flwr new  # Create a new Flower app from a template"
+        "\n\n\t\t$ flwr run  # Run the Flower app in Simulation Mode"
+        "\n\n\tUsing `start_simulation()` is deprecated."
+    )
+    warn_deprecated_feature(name=msg)
+
+    event(
+        EventType.START_SIMULATION_ENTER,
+        {"num_clients": len(clients_ids) if clients_ids is not None else num_clients},
+    )
+
+    if clients_ids is not None:
+        warn_unsupported_feature(
+            "Passing `clients_ids` to `start_simulation` is deprecated and not longer "
+            "used by `start_simulation`. Use `num_clients` exclusively instead."
+        )
+        log(ERROR, "`clients_ids` argument used.")
+        sys.exit()
+
+    # Set logger propagation
+    loop: Optional[asyncio.AbstractEventLoop] = None
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        loop = None
+    finally:
+        if loop and loop.is_running():
+            # Set logger propagation to False to prevent duplicated log output in Colab.
+            logger = logging.getLogger("flwr")
+            _ = set_logger_propagation(logger, False)
+
+    # Initialize server and server config
+    initialized_server, initialized_config = init_defaults(
+        server=server,
+        config=config,
+        strategy=strategy,
+        client_manager=client_manager,
+    )
+
+    log(
+        INFO,
+        "Starting Flower simulation, config: %s",
+        initialized_config,
+    )
+
+    # Create node-id to partition-id mapping
+    nodes_mapping = _create_node_id_to_partition_mapping(num_clients)
+
+    # Default arguments for Ray initialization
+    if not ray_init_args:
+        ray_init_args = {
+            "ignore_reinit_error": True,
+            "include_dashboard": False,
+        }
+
+    # Shut down Ray if it has already been initialized (unless asked not to)
+    if ray.is_initialized() and not keep_initialised:
+        ray.shutdown()
+
+    # Initialize Ray
+    ray.init(**ray_init_args)
+    cluster_resources = ray.cluster_resources()
+    log(
+        INFO,
+        "Flower VCE: Ray initialized with resources: %s",
+        cluster_resources,
+    )
+
+    log(
+        INFO,
+        "Optimize your simulation with Flower VCE: "
+        "https://flower.ai/docs/framework/how-to-run-simulations.html",
+    )
+
+    # Log the resources that a single client will be able to use
+    if client_resources is None:
+        log(
+            INFO,
+            "No `client_resources` specified. Using minimal resources for clients.",
+        )
+        client_resources = {"num_cpus": 1, "num_gpus": 0.0}
+
+    # Each client needs at the very least one CPU
+    if "num_cpus" not in client_resources:
+        warnings.warn(
+            "No `num_cpus` specified in `client_resources`. "
+            "Using `num_cpus=1` for each client.",
+            stacklevel=2,
+        )
+        client_resources["num_cpus"] = 1
+
+    log(
+        INFO,
+        "Flower VCE: Resources for each Virtual Client: %s",
+        client_resources,
+    )
+
+    actor_args = {} if actor_kwargs is None else actor_kwargs
+
+    # An actor factory. This is called N times to add N actors
+    # to the pool. If at some point the pool can accommodate more actors
+    # this will be called again.
+    def create_actor_fn() -> type[VirtualClientEngineActor]:
+        return actor_type.options(  # type: ignore
+            **client_resources,
+            scheduling_strategy=actor_scheduling,
+        ).remote(**actor_args)
+
+    # Instantiate ActorPool
+    pool = VirtualClientEngineActorPool(
+        create_actor_fn=create_actor_fn,
+        client_resources=client_resources,
+    )
+
+    f_stop = threading.Event()
+
+    # Periodically, check if the cluster has grown (i.e. a new
+    # node has been added). If this happens, we likely want to grow
+    # the actor pool by adding more Actors to it.
+    def update_resources(f_stop: threading.Event) -> None:
+        """Periodically check if more actors can be added to the pool.
+
+        If so, extend the pool.
+        """
+        if not f_stop.is_set():
+            num_max_actors = pool_size_from_resources(client_resources)
+            if num_max_actors > pool.num_actors:
+                num_new = num_max_actors - pool.num_actors
+                log(
+                    INFO, "The cluster expanded. Adding %s actors to the pool.", num_new
+                )
+                pool.add_actors_to_pool(num_actors=num_new)
+
+            threading.Timer(10, update_resources, [f_stop]).start()
+
+    update_resources(f_stop)
+
+    log(
+        INFO,
+        "Flower VCE: Creating %s with %s actors",
+        pool.__class__.__name__,
+        pool.num_actors,
+    )
+
+    # Register one RayClientProxy object for each client with the ClientManager
+    for node_id, partition_id in nodes_mapping.items():
+        client_proxy = RayActorClientProxy(
+            client_fn=client_fn,
+            node_id=node_id,
+            partition_id=partition_id,
+            num_partitions=num_clients,
+            actor_pool=pool,
+        )
+        initialized_server.client_manager().register(client=client_proxy)
+
+    hist = History()
+    # pylint: disable=broad-except
+    try:
+        # Start training
+        hist = run_fl(
+            server=initialized_server,
+            config=initialized_config,
+        )
+    except Exception as ex:
+        log(ERROR, ex)
+        log(ERROR, traceback.format_exc())
+        log(
+            ERROR,
+            "Your simulation crashed :(. This could be because of several reasons. "
+            "The most common are: "
+            "\n\t > Sometimes, issues in the simulation code itself can cause crashes. "
+            "It's always a good idea to double-check your code for any potential bugs "
+            "or inconsistencies that might be contributing to the problem. "
+            "For example: "
+            "\n\t\t - You might be using a class attribute in your clients that "
+            "hasn't been defined."
+            "\n\t\t - There could be an incorrect method call to a 3rd party library "
+            "(e.g., PyTorch)."
+            "\n\t\t - The return types of methods in your clients/strategies might be "
+            "incorrect."
+            "\n\t > Your system couldn't fit a single VirtualClient: try lowering "
+            "`client_resources`."
+            "\n\t > All the actors in your pool crashed. This could be because: "
+            "\n\t\t - You clients hit an out-of-memory (OOM) error and actors couldn't "
+            "recover from it. Try launching your simulation with more generous "
+            "`client_resources` setting (i.e. it seems %s is "
+            "not enough for your run). Use fewer concurrent actors. "
+            "\n\t\t - You were running a multi-node simulation and all worker nodes "
+            "disconnected. The head node might still be alive but cannot accommodate "
+            "any actor with resources: %s."
+            "\nTake a look at the Flower simulation examples for guidance "
+            "<https://flower.ai/docs/framework/how-to-run-simulations.html>.",
+            client_resources,
+            client_resources,
+        )
+        raise RuntimeError("Simulation crashed.") from ex
+
+    finally:
+        # Stop time monitoring resources in cluster
+        f_stop.set()
+        event(EventType.START_SIMULATION_LEAVE)
+
+    return hist

flwr/simulation/ray_transport/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Flower Labs GmbH. All Rights Reserved.
+# Copyright 2021 Flower Labs GmbH. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

flwr/simulation/ray_transport/ray_actor.py

@@ -14,33 +14,23 @@
 # ==============================================================================
 """Ray-based Flower Actor and ActorPool implementation."""
 
-import asyncio
+
 import threading
-import traceback
 from abc import ABC
 from logging import DEBUG, ERROR, WARNING
-from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
+from typing import Any, Callable, Optional, Union
 
 import ray
 from ray import ObjectRef
 from ray.util.actor_pool import ActorPool
 
-from flwr.client.client_app import ClientApp, LoadClientAppError
+from flwr.client.client_app import ClientApp, ClientAppException, LoadClientAppError
 from flwr.common import Context, Message
 from flwr.common.logger import log
 
 ClientAppFn = Callable[[], ClientApp]
 
 
-class ClientException(Exception):
-    """Raised when client side logic crashes with an exception."""
-
-    def __init__(self, message: str):
-        div = ">" * 7
-        self.message = "\n" + div + "A ClientException occurred." + message
-        super().__init__(self.message)
-
-
 class VirtualClientEngineActor(ABC):
     """Abstract base class for VirtualClientEngine Actors."""
 
@@ -55,7 +45,7 @@ class VirtualClientEngineActor(ABC):
         message: Message,
         cid: str,
         context: Context,
-    ) -> Tuple[str, Message, Context]:
+    ) -> tuple[str, Message, Context]:
         """Run a client run."""
         # Pass message through ClientApp and return a message
         # return also cid which is needed to ensure results
@@ -71,17 +61,7 @@ class VirtualClientEngineActor(ABC):
             raise load_ex
 
         except Exception as ex:
-            client_trace = traceback.format_exc()
-            mssg = (
-                "\n\tSomething went wrong when running your client run."
-                "\n\tClient "
-                + cid
-                + " crashed when the "
-                + self.__class__.__name__
-                + " was running its run."
-                "\n\tException triggered on the client side: " + client_trace,
-            )
-            raise ClientException(str(mssg)) from ex
+            raise ClientAppException(str(ex)) from ex
 
         return cid, out_message, context
 
@@ -102,7 +82,7 @@ class ClientAppActor(VirtualClientEngineActor):
             on_actor_init_fn()
 
 
-def pool_size_from_resources(client_resources: Dict[str, Union[int, float]]) -> int:
+def pool_size_from_resources(client_resources: dict[str, Union[int, float]]) -> int:
     """Calculate number of Actors that fit in the cluster.
 
     For this we consider the resources available on each node and those required per
@@ -145,14 +125,14 @@ def pool_size_from_resources(client_resources: Dict[str, Union[int, float]]) ->
             WARNING,
             "The ActorPool is empty. The system (CPUs=%s, GPUs=%s) "
             "does not meet the criteria to host at least one client with resources:"
-            " %s. Lowering the `client_resources` could help.",
+            " %s. Lowering these resources could help.",
             num_cpus,
             num_gpus,
             client_resources,
         )
         raise ValueError(
             "ActorPool is empty. Stopping Simulation. "
-            "Check 'client_resources' passed to `start_simulation`"
+            "Check `num_cpus` and/or `num_gpus` passed to the simulation engine"
         )
 
     return total_num_actors
@@ -183,9 +163,9 @@ class VirtualClientEngineActorPool(ActorPool):
 
     def __init__(
         self,
-        create_actor_fn: Callable[[], Type[VirtualClientEngineActor]],
-        client_resources: Dict[str, Union[int, float]],
-        actor_list: Optional[List[Type[VirtualClientEngineActor]]] = None,
+        create_actor_fn: Callable[[], type[VirtualClientEngineActor]],
+        client_resources: dict[str, Union[int, float]],
+        actor_list: Optional[list[type[VirtualClientEngineActor]]] = None,
     ):
         self.client_resources = client_resources
         self.create_actor_fn = create_actor_fn
@@ -204,10 +184,10 @@ class VirtualClientEngineActorPool(ActorPool):
 
         # A dict that maps cid to another dict containing: a reference to the remote job
         # and its status (i.e. whether it is ready or not)
-        self._cid_to_future: Dict[
-            str, Dict[str, Union[bool, Optional[ObjectRef[Any]]]]
+        self._cid_to_future: dict[
+            str, dict[str, Union[bool, Optional[ObjectRef[Any]]]]
         ] = {}
-        self.actor_to_remove: Set[str] = set()  # a set
+        self.actor_to_remove: set[str] = set()  # a set
         self.num_actors = len(actors)
 
         self.lock = threading.RLock()
@@ -231,7 +211,7 @@ class VirtualClientEngineActorPool(ActorPool):
             self._idle_actors.extend(new_actors)
             self.num_actors += num_actors
 
-    def submit(self, fn: Any, value: Tuple[ClientAppFn, Message, str, Context]) -> None:
+    def submit(self, fn: Any, value: tuple[ClientAppFn, Message, str, Context]) -> None:
         """Take an idle actor and assign it to run a client app and Message.
 
         Submit a job to an actor by first removing it from the list of idle actors, then
@@ -241,7 +221,7 @@ class VirtualClientEngineActorPool(ActorPool):
         actor = self._idle_actors.pop()
         if self._check_and_remove_actor_from_pool(actor):
             future = fn(actor, app_fn, mssg, cid, context)
-            future_key = tuple(future) if isinstance(future, List) else future
+            future_key = tuple(future) if isinstance(future, list) else future
             self._future_to_actor[future_key] = (self._next_task_index, actor, cid)
             self._next_task_index += 1
 
@@ -249,7 +229,7 @@ class VirtualClientEngineActorPool(ActorPool):
             self._cid_to_future[cid]["future"] = future_key
 
     def submit_client_job(
-        self, actor_fn: Any, job: Tuple[ClientAppFn, Message, str, Context]
+        self, actor_fn: Any, job: tuple[ClientAppFn, Message, str, Context]
     ) -> None:
         """Submit a job while tracking client ids."""
         _, _, cid, _ = job
@@ -289,7 +269,7 @@ class VirtualClientEngineActorPool(ActorPool):
 
         return self._cid_to_future[cid]["ready"]  # type: ignore
 
-    def _fetch_future_result(self, cid: str) -> Tuple[Message, Context]:
+    def _fetch_future_result(self, cid: str) -> tuple[Message, Context]:
         """Fetch result and updated context for a VirtualClient from Object Store.
 
         The job submitted by the ClientProxy interfacing with client with cid=cid is
@@ -403,7 +383,7 @@ class VirtualClientEngineActorPool(ActorPool):
 
     def get_client_result(
         self, cid: str, timeout: Optional[float]
-    ) -> Tuple[Message, Context]:
+    ) -> tuple[Message, Context]:
         """Get result from VirtualClient with specific cid."""
         # Loop until all jobs submitted to the pool are completed. Break early
         # if the result for the ClientProxy calling this method is ready
@@ -419,27 +399,19 @@ class VirtualClientEngineActorPool(ActorPool):
         return self._fetch_future_result(cid)
 
 
-def init_ray(*args: Any, **kwargs: Any) -> None:
-    """Intialises Ray if not already initialised."""
-    if not ray.is_initialized():
-        ray.init(*args, **kwargs)
-
-
 class BasicActorPool:
     """A basic actor pool."""
 
     def __init__(
         self,
-        actor_type: Type[VirtualClientEngineActor],
-        client_resources: Dict[str, Union[int, float]],
-        actor_kwargs: Dict[str, Any],
+        actor_type: type[VirtualClientEngineActor],
+        client_resources: dict[str, Union[int, float]],
+        actor_kwargs: dict[str, Any],
     ):
         self.client_resources = client_resources
 
         # Queue of idle actors
-        self.pool: "asyncio.Queue[Type[VirtualClientEngineActor]]" = asyncio.Queue(
-            maxsize=1024
-        )
+        self.pool: list[VirtualClientEngineActor] = []
         self.num_actors = 0
 
         # Resolve arguments to pass during actor init
@@ -453,38 +425,37 @@ class BasicActorPool:
         # Figure out how many actors can be created given the cluster resources
         # and the resources the user indicates each VirtualClient will need
         self.actors_capacity = pool_size_from_resources(client_resources)
-        self._future_to_actor: Dict[Any, Type[VirtualClientEngineActor]] = {}
+        self._future_to_actor: dict[Any, VirtualClientEngineActor] = {}
 
     def is_actor_available(self) -> bool:
         """Return true if there is an idle actor."""
-        return self.pool.qsize() > 0
+        return len(self.pool) > 0
 
-    async def add_actors_to_pool(self, num_actors: int) -> None:
+    def add_actors_to_pool(self, num_actors: int) -> None:
         """Add actors to the pool.
 
         This method may be executed also if new resources are added to your Ray cluster
         (e.g. you add a new node).
         """
         for _ in range(num_actors):
-            await self.pool.put(self.create_actor_fn())  # type: ignore
+            self.pool.append(self.create_actor_fn())  # type: ignore
         self.num_actors += num_actors
 
-    async def terminate_all_actors(self) -> None:
+    def terminate_all_actors(self) -> None:
         """Terminate actors in pool."""
         num_terminated = 0
-        while self.pool.qsize():
-            actor = await self.pool.get()
+        for actor in self.pool:
             actor.terminate.remote()  # type: ignore
             num_terminated += 1
 
         log(DEBUG, "Terminated %i actors", num_terminated)
 
-    async def submit(
-        self, actor_fn: Any, job: Tuple[ClientAppFn, Message, str, Context]
+    def submit(
+        self, actor_fn: Any, job: tuple[ClientAppFn, Message, str, Context]
     ) -> Any:
         """On idle actor, submit job and return future."""
         # Remove idle actor from pool
-        actor = await self.pool.get()
+        actor = self.pool.pop()
         # Submit job to actor
         app_fn, mssg, cid, context = job
         future = actor_fn(actor, app_fn, mssg, cid, context)
@@ -493,14 +464,18 @@ class BasicActorPool:
         self._future_to_actor[future] = actor
         return future
 
-    async def fetch_result_and_return_actor_to_pool(
+    def add_actor_back_to_pool(self, future: Any) -> None:
+        """Ad actor assigned to run future back into the pool."""
+        actor = self._future_to_actor.pop(future)
+        self.pool.append(actor)
+
+    def fetch_result_and_return_actor_to_pool(
         self, future: Any
-    ) -> Tuple[Message, Context]:
+    ) -> tuple[Message, Context]:
         """Pull result given a future and add actor back to pool."""
-        # Get actor that ran job
-        actor = self._future_to_actor.pop(future)
-        await self.pool.put(actor)
         # Retrieve result for object store
         # Instead of doing ray.get(future) we await it
-        _, out_mssg, updated_context = await future
+        _, out_mssg, updated_context = ray.get(future)
+        # Get actor that ran job
+        self.add_actor_back_to_pool(future)
         return out_mssg, updated_context