flwr-nightly 1.11.0.dev20240822__py3-none-any.whl → 1.11.1.dev20240912__py3-none-any.whl

flwr/server/superlink/fleet/grpc_adapter/grpc_adapter_servicer.py

@@ -23,6 +23,7 @@ from google.protobuf.message import Message as GrpcMessage
 
 from flwr.common.logger import log
 from flwr.proto import grpcadapter_pb2_grpc  # pylint: disable=E0611
+from flwr.proto.fab_pb2 import GetFabRequest, GetFabResponse  # pylint: disable=E0611
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     CreateNodeResponse,
@@ -37,6 +38,7 @@ from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
 )
 from flwr.proto.grpcadapter_pb2 import MessageContainer  # pylint: disable=E0611
 from flwr.proto.run_pb2 import GetRunRequest, GetRunResponse  # pylint: disable=E0611
+from flwr.server.superlink.ffs.ffs_factory import FfsFactory
 from flwr.server.superlink.fleet.message_handler import message_handler
 from flwr.server.superlink.state import StateFactory
 
@@ -60,10 +62,11 @@ def _handle(
 class GrpcAdapterServicer(grpcadapter_pb2_grpc.GrpcAdapterServicer):
     """Fleet API via GrpcAdapter servicer."""
 
-    def __init__(self, state_factory: StateFactory) -> None:
+    def __init__(self, state_factory: StateFactory, ffs_factory: FfsFactory) -> None:
         self.state_factory = state_factory
+        self.ffs_factory = ffs_factory
 
-    def SendReceive(
+    def SendReceive(  # pylint: disable=too-many-return-statements
         self, request: MessageContainer, context: grpc.ServicerContext
     ) -> MessageContainer:
         """."""
@@ -80,6 +83,8 @@ class GrpcAdapterServicer(grpcadapter_pb2_grpc.GrpcAdapterServicer):
             return _handle(request, PushTaskResRequest, self._push_task_res)
         if request.grpc_message_name == GetRunRequest.__qualname__:
             return _handle(request, GetRunRequest, self._get_run)
+        if request.grpc_message_name == GetFabRequest.__qualname__:
+            return _handle(request, GetFabRequest, self._get_fab)
         raise ValueError(f"Invalid grpc_message_name: {request.grpc_message_name}")
 
     def _create_node(self, request: CreateNodeRequest) -> CreateNodeResponse:
@@ -129,3 +134,11 @@ class GrpcAdapterServicer(grpcadapter_pb2_grpc.GrpcAdapterServicer):
             request=request,
             state=self.state_factory.state(),
         )
+
+    def _get_fab(self, request: GetFabRequest) -> GetFabResponse:
+        """Get FAB."""
+        log(INFO, "GrpcAdapter.GetFab")
+        return message_handler.get_fab(
+            request=request,
+            ffs=self.ffs_factory.ffs(),
+        )

flwr/server/superlink/fleet/grpc_bidi/grpc_server.py

@@ -23,6 +23,7 @@ from typing import Any, Callable, Optional, Sequence, Tuple, Union
 import grpc
 
 from flwr.common import GRPC_MAX_MESSAGE_LENGTH
+from flwr.common.address import is_port_in_use
 from flwr.common.logger import log
 from flwr.proto.transport_pb2_grpc import (  # pylint: disable=E0611
     add_FlowerServiceServicer_to_server,
@@ -218,6 +219,10 @@ def generic_create_grpc_server(  # pylint: disable=too-many-arguments
     server : grpc.Server
         A non-running instance of a gRPC server.
     """
+    # Check if port is in use
+    if is_port_in_use(server_address):
+        sys.exit(f"Port in server address {server_address} is already in use.")
+
     # Deconstruct tuple into servicer and function
     servicer, add_servicer_to_server_fn = servicer_and_add_fn
 

flwr/server/superlink/fleet/grpc_rere/fleet_servicer.py

@@ -51,19 +51,22 @@ class FleetServicer(fleet_pb2_grpc.FleetServicer):
         self, request: CreateNodeRequest, context: grpc.ServicerContext
     ) -> CreateNodeResponse:
         """."""
-        log(INFO, "FleetServicer.CreateNode")
+        log(INFO, "[Fleet.CreateNode] Request ping_interval=%s", request.ping_interval)
+        log(DEBUG, "[Fleet.CreateNode] Request: %s", request)
         response = message_handler.create_node(
             request=request,
             state=self.state_factory.state(),
         )
-        log(INFO, "FleetServicer: Created node_id=%s", response.node.node_id)
+        log(INFO, "[Fleet.CreateNode] Created node_id=%s", response.node.node_id)
+        log(DEBUG, "[Fleet.CreateNode] Response: %s", response)
         return response
 
     def DeleteNode(
         self, request: DeleteNodeRequest, context: grpc.ServicerContext
     ) -> DeleteNodeResponse:
         """."""
-        log(INFO, "FleetServicer.DeleteNode")
+        log(INFO, "[Fleet.DeleteNode] Delete node_id=%s", request.node.node_id)
+        log(DEBUG, "[Fleet.DeleteNode] Request: %s", request)
         return message_handler.delete_node(
             request=request,
             state=self.state_factory.state(),
@@ -71,7 +74,7 @@ class FleetServicer(fleet_pb2_grpc.FleetServicer):
 
     def Ping(self, request: PingRequest, context: grpc.ServicerContext) -> PingResponse:
         """."""
-        log(DEBUG, "FleetServicer.Ping")
+        log(DEBUG, "[Fleet.Ping] Request: %s", request)
         return message_handler.ping(
             request=request,
             state=self.state_factory.state(),
@@ -81,7 +84,8 @@ class FleetServicer(fleet_pb2_grpc.FleetServicer):
         self, request: PullTaskInsRequest, context: grpc.ServicerContext
     ) -> PullTaskInsResponse:
         """Pull TaskIns."""
-        log(INFO, "FleetServicer.PullTaskIns")
+        log(INFO, "[Fleet.PullTaskIns] node_id=%s", request.node.node_id)
+        log(DEBUG, "[Fleet.PullTaskIns] Request: %s", request)
         return message_handler.pull_task_ins(
             request=request,
             state=self.state_factory.state(),
@@ -91,7 +95,14 @@ class FleetServicer(fleet_pb2_grpc.FleetServicer):
         self, request: PushTaskResRequest, context: grpc.ServicerContext
     ) -> PushTaskResResponse:
         """Push TaskRes."""
-        log(INFO, "FleetServicer.PushTaskRes")
+        if request.task_res_list:
+            log(
+                INFO,
+                "[Fleet.PushTaskRes] Push results from node_id=%s",
+                request.task_res_list[0].task.producer.node_id,
+            )
+        else:
+            log(INFO, "[Fleet.PushTaskRes] No task results to push")
         return message_handler.push_task_res(
             request=request,
             state=self.state_factory.state(),
@@ -101,7 +112,7 @@ class FleetServicer(fleet_pb2_grpc.FleetServicer):
         self, request: GetRunRequest, context: grpc.ServicerContext
     ) -> GetRunResponse:
         """Get run information."""
-        log(INFO, "FleetServicer.GetRun")
+        log(INFO, "[Fleet.GetRun] Requesting `Run` for run_id=%s", request.run_id)
         return message_handler.get_run(
             request=request,
             state=self.state_factory.state(),
@@ -111,7 +122,7 @@ class FleetServicer(fleet_pb2_grpc.FleetServicer):
         self, request: GetFabRequest, context: grpc.ServicerContext
     ) -> GetFabResponse:
         """Get FAB."""
-        log(DEBUG, "DriverServicer.GetFab")
+        log(INFO, "[Fleet.GetFab] Requesting FAB for fab_hash=%s", request.hash_str)
         return message_handler.get_fab(
             request=request,
             ffs=self.ffs_factory.ffs(),

flwr/server/superlink/fleet/grpc_rere/server_interceptor.py

@@ -78,13 +78,13 @@ def _get_value_from_tuples(
 
 
 class AuthenticateServerInterceptor(grpc.ServerInterceptor):  # type: ignore
-    """Server interceptor for client authentication."""
+    """Server interceptor for node authentication."""
 
     def __init__(self, state: State):
         self.state = state
 
-        self.client_public_keys = state.get_client_public_keys()
-        if len(self.client_public_keys) == 0:
+        self.node_public_keys = state.get_node_public_keys()
+        if len(self.node_public_keys) == 0:
             log(WARNING, "Authentication enabled, but no known public keys configured")
 
         private_key = self.state.get_server_private_key()
@@ -103,9 +103,9 @@ class AuthenticateServerInterceptor(grpc.ServerInterceptor):  # type: ignore
     ) -> grpc.RpcMethodHandler:
         """Flower server interceptor authentication logic.
 
-        Intercept all unary calls from clients and authenticate clients by validating
-        auth metadata sent by the client. Continue RPC call if client is authenticated,
-        else, terminate RPC call by setting context to abort.
+        Intercept all unary calls from nodes and authenticate nodes by validating auth
+        metadata sent by the node. Continue RPC call if node is authenticated, else,
+        terminate RPC call by setting context to abort.
         """
         # One of the method handlers in
         # `flwr.server.superlink.fleet.grpc_rere.fleet_server.FleetServicer`
@@ -119,17 +119,17 @@ class AuthenticateServerInterceptor(grpc.ServerInterceptor):  # type: ignore
             request: Request,
             context: grpc.ServicerContext,
         ) -> Response:
-            client_public_key_bytes = base64.urlsafe_b64decode(
+            node_public_key_bytes = base64.urlsafe_b64decode(
                 _get_value_from_tuples(
                     _PUBLIC_KEY_HEADER, context.invocation_metadata()
                 )
             )
-            if client_public_key_bytes not in self.client_public_keys:
+            if node_public_key_bytes not in self.node_public_keys:
                 context.abort(grpc.StatusCode.UNAUTHENTICATED, "Access denied")
 
             if isinstance(request, CreateNodeRequest):
                 response = self._create_authenticated_node(
-                    client_public_key_bytes, request, context
+                    node_public_key_bytes, request, context
                 )
                 log(
                     INFO,
@@ -144,13 +144,13 @@ class AuthenticateServerInterceptor(grpc.ServerInterceptor):  # type: ignore
                     _AUTH_TOKEN_HEADER, context.invocation_metadata()
                 )
             )
-            public_key = bytes_to_public_key(client_public_key_bytes)
+            public_key = bytes_to_public_key(node_public_key_bytes)
 
             if not self._verify_hmac(public_key, request, hmac_value):
                 context.abort(grpc.StatusCode.UNAUTHENTICATED, "Access denied")
 
             # Verify node_id
-            node_id = self.state.get_node_id(client_public_key_bytes)
+            node_id = self.state.get_node_id(node_public_key_bytes)
 
             if not self._verify_node_id(node_id, request):
                 context.abort(grpc.StatusCode.UNAUTHENTICATED, "Access denied")
@@ -188,7 +188,8 @@ class AuthenticateServerInterceptor(grpc.ServerInterceptor):  # type: ignore
         self, public_key: ec.EllipticCurvePublicKey, request: Request, hmac_value: bytes
     ) -> bool:
         shared_secret = generate_shared_key(self.server_private_key, public_key)
-        return verify_hmac(shared_secret, request.SerializeToString(True), hmac_value)
+        message_bytes = request.SerializeToString(deterministic=True)
+        return verify_hmac(shared_secret, message_bytes, hmac_value)
 
     def _create_authenticated_node(
         self,

flwr/server/superlink/fleet/rest_rere/rest_api.py

@@ -15,17 +15,29 @@
 """Experimental REST API server."""
 
 
+from __future__ import annotations
+
 import sys
+from typing import Awaitable, Callable, TypeVar
+
+from google.protobuf.message import Message as GrpcMessage
 
 from flwr.common.constant import MISSING_EXTRA_REST
+from flwr.proto.fab_pb2 import GetFabRequest, GetFabResponse  # pylint: disable=E0611
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
+    CreateNodeResponse,
     DeleteNodeRequest,
+    DeleteNodeResponse,
     PingRequest,
+    PingResponse,
     PullTaskInsRequest,
+    PullTaskInsResponse,
     PushTaskResRequest,
+    PushTaskResResponse,
 )
-from flwr.proto.run_pb2 import GetRunRequest  # pylint: disable=E0611
+from flwr.proto.run_pb2 import GetRunRequest, GetRunResponse  # pylint: disable=E0611
+from flwr.server.superlink.ffs.ffs import Ffs
 from flwr.server.superlink.fleet.message_handler import message_handler
 from flwr.server.superlink.state import State
 
@@ -40,172 +52,108 @@ except ModuleNotFoundError:
     sys.exit(MISSING_EXTRA_REST)
 
 
-async def create_node(request: Request) -> Response:
-    """Create Node."""
-    _check_headers(request.headers)
+GrpcRequest = TypeVar("GrpcRequest", bound=GrpcMessage)
+GrpcResponse = TypeVar("GrpcResponse", bound=GrpcMessage)
 
-    # Get the request body as raw bytes
-    create_node_request_bytes: bytes = await request.body()
+GrpcAsyncFunction = Callable[[GrpcRequest], Awaitable[GrpcResponse]]
+RestEndPoint = Callable[[Request], Awaitable[Response]]
 
-    # Deserialize ProtoBuf
-    create_node_request_proto = CreateNodeRequest()
-    create_node_request_proto.ParseFromString(create_node_request_bytes)
 
-    # Get state from app
-    state: State = app.state.STATE_FACTORY.state()
+def rest_request_response(
+    grpc_request_type: type[GrpcRequest],
+) -> Callable[[GrpcAsyncFunction[GrpcRequest, GrpcResponse]], RestEndPoint]:
+    """Convert an async gRPC-based function into a RESTful HTTP endpoint."""
 
-    # Handle message
-    create_node_response_proto = message_handler.create_node(
-        request=create_node_request_proto, state=state
-    )
+    def decorator(func: GrpcAsyncFunction[GrpcRequest, GrpcResponse]) -> RestEndPoint:
+        async def wrapper(request: Request) -> Response:
+            _check_headers(request.headers)
 
-    # Return serialized ProtoBuf
-    create_node_response_bytes = create_node_response_proto.SerializeToString()
-    return Response(
-        status_code=200,
-        content=create_node_response_bytes,
-        headers={"Content-Type": "application/protobuf"},
-    )
+            # Get the request body as raw bytes
+            grpc_req_bytes: bytes = await request.body()
 
+            # Deserialize ProtoBuf
+            grpc_req = grpc_request_type.FromString(grpc_req_bytes)
+            grpc_res = await func(grpc_req)
+            return Response(
+                status_code=200,
+                content=grpc_res.SerializeToString(),
+                headers={"Content-Type": "application/protobuf"},
+            )
 
-async def delete_node(request: Request) -> Response:
-    """Delete Node Id."""
-    _check_headers(request.headers)
+        return wrapper
 
-    # Get the request body as raw bytes
-    delete_node_request_bytes: bytes = await request.body()
+    return decorator
 
-    # Deserialize ProtoBuf
-    delete_node_request_proto = DeleteNodeRequest()
-    delete_node_request_proto.ParseFromString(delete_node_request_bytes)
 
+@rest_request_response(CreateNodeRequest)
+async def create_node(request: CreateNodeRequest) -> CreateNodeResponse:
+    """Create Node."""
     # Get state from app
     state: State = app.state.STATE_FACTORY.state()
 
     # Handle message
-    delete_node_response_proto = message_handler.delete_node(
-        request=delete_node_request_proto, state=state
-    )
+    return message_handler.create_node(request=request, state=state)
 
-    # Return serialized ProtoBuf
-    delete_node_response_bytes = delete_node_response_proto.SerializeToString()
-    return Response(
-        status_code=200,
-        content=delete_node_response_bytes,
-        headers={"Content-Type": "application/protobuf"},
-    )
 
+@rest_request_response(DeleteNodeRequest)
+async def delete_node(request: DeleteNodeRequest) -> DeleteNodeResponse:
+    """Delete Node Id."""
+    # Get state from app
+    state: State = app.state.STATE_FACTORY.state()
 
-async def pull_task_ins(request: Request) -> Response:
-    """Pull TaskIns."""
-    _check_headers(request.headers)
-
-    # Get the request body as raw bytes
-    pull_task_ins_request_bytes: bytes = await request.body()
+    # Handle message
+    return message_handler.delete_node(request=request, state=state)
 
-    # Deserialize ProtoBuf
-    pull_task_ins_request_proto = PullTaskInsRequest()
-    pull_task_ins_request_proto.ParseFromString(pull_task_ins_request_bytes)
 
+@rest_request_response(PullTaskInsRequest)
+async def pull_task_ins(request: PullTaskInsRequest) -> PullTaskInsResponse:
+    """Pull TaskIns."""
     # Get state from app
     state: State = app.state.STATE_FACTORY.state()
 
     # Handle message
-    pull_task_ins_response_proto = message_handler.pull_task_ins(
-        request=pull_task_ins_request_proto,
-        state=state,
-    )
-
-    # Return serialized ProtoBuf
-    pull_task_ins_response_bytes = pull_task_ins_response_proto.SerializeToString()
-    return Response(
-        status_code=200,
-        content=pull_task_ins_response_bytes,
-        headers={"Content-Type": "application/protobuf"},
-    )
+    return message_handler.pull_task_ins(request=request, state=state)
 
 
-async def push_task_res(request: Request) -> Response:  # Check if token is needed here
+# Check if token is needed here
+@rest_request_response(PushTaskResRequest)
+async def push_task_res(request: PushTaskResRequest) -> PushTaskResResponse:
     """Push TaskRes."""
-    _check_headers(request.headers)
-
-    # Get the request body as raw bytes
-    push_task_res_request_bytes: bytes = await request.body()
-
-    # Deserialize ProtoBuf
-    push_task_res_request_proto = PushTaskResRequest()
-    push_task_res_request_proto.ParseFromString(push_task_res_request_bytes)
-
     # Get state from app
     state: State = app.state.STATE_FACTORY.state()
 
     # Handle message
-    push_task_res_response_proto = message_handler.push_task_res(
-        request=push_task_res_request_proto,
-        state=state,
-    )
-
-    # Return serialized ProtoBuf
-    push_task_res_response_bytes = push_task_res_response_proto.SerializeToString()
-    return Response(
-        status_code=200,
-        content=push_task_res_response_bytes,
-        headers={"Content-Type": "application/protobuf"},
-    )
+    return message_handler.push_task_res(request=request, state=state)
 
 
-async def ping(request: Request) -> Response:
+@rest_request_response(PingRequest)
+async def ping(request: PingRequest) -> PingResponse:
     """Ping."""
-    _check_headers(request.headers)
-
-    # Get the request body as raw bytes
-    ping_request_bytes: bytes = await request.body()
-
-    # Deserialize ProtoBuf
-    ping_request_proto = PingRequest()
-    ping_request_proto.ParseFromString(ping_request_bytes)
-
     # Get state from app
     state: State = app.state.STATE_FACTORY.state()
 
     # Handle message
-    ping_response_proto = message_handler.ping(request=ping_request_proto, state=state)
-
-    # Return serialized ProtoBuf
-    ping_response_bytes = ping_response_proto.SerializeToString()
-    return Response(
-        status_code=200,
-        content=ping_response_bytes,
-        headers={"Content-Type": "application/protobuf"},
-    )
+    return message_handler.ping(request=request, state=state)
 
 
-async def get_run(request: Request) -> Response:
+@rest_request_response(GetRunRequest)
+async def get_run(request: GetRunRequest) -> GetRunResponse:
     """GetRun."""
-    _check_headers(request.headers)
-
-    # Get the request body as raw bytes
-    get_run_request_bytes: bytes = await request.body()
-
-    # Deserialize ProtoBuf
-    get_run_request_proto = GetRunRequest()
-    get_run_request_proto.ParseFromString(get_run_request_bytes)
-
     # Get state from app
     state: State = app.state.STATE_FACTORY.state()
 
     # Handle message
-    get_run_response_proto = message_handler.get_run(
-        request=get_run_request_proto, state=state
-    )
+    return message_handler.get_run(request=request, state=state)
+
 
-    # Return serialized ProtoBuf
-    get_run_response_bytes = get_run_response_proto.SerializeToString()
-    return Response(
-        status_code=200,
-        content=get_run_response_bytes,
-        headers={"Content-Type": "application/protobuf"},
-    )
+@rest_request_response(GetFabRequest)
+async def get_fab(request: GetFabRequest) -> GetFabResponse:
+    """GetRun."""
+    # Get ffs from app
+    ffs: Ffs = app.state.FFS_FACTORY.state()
+
+    # Handle message
+    return message_handler.get_fab(request=request, ffs=ffs)
 
 
 routes = [
@@ -215,6 +163,7 @@ routes = [
     Route("/api/v0/fleet/push-task-res", push_task_res, methods=["POST"]),
     Route("/api/v0/fleet/ping", ping, methods=["POST"]),
     Route("/api/v0/fleet/get-run", get_run, methods=["POST"]),
+    Route("/api/v0/fleet/get-fab", get_fab, methods=["POST"]),
 ]
 
 app: Starlette = Starlette(

flwr/server/superlink/fleet/vce/backend/backend.py

@@ -33,7 +33,7 @@ class Backend(ABC):
         """Construct a backend."""
 
     @abstractmethod
-    def build(self) -> None:
+    def build(self, app_fn: Callable[[], ClientApp]) -> None:
         """Build backend.
 
         Different components need to be in place before workers in a backend are ready
@@ -60,7 +60,6 @@ class Backend(ABC):
     @abstractmethod
     def process_message(
         self,
-        app: Callable[[], ClientApp],
         message: Message,
         context: Context,
     ) -> Tuple[Message, Context]:

flwr/server/superlink/fleet/vce/backend/raybackend.py

@@ -16,7 +16,7 @@
 
 import sys
 from logging import DEBUG, ERROR
-from typing import Callable, Dict, Tuple, Union
+from typing import Callable, Dict, Optional, Tuple, Union
 
 import ray
 
@@ -52,16 +52,13 @@ class RayBackend(Backend):
 
         # Validate client resources
         self.client_resources_key = "client_resources"
-        client_resources = self._validate_client_resources(config=backend_config)
+        self.client_resources = self._validate_client_resources(config=backend_config)
 
-        # Create actor pool
-        actor_kwargs = self._validate_actor_arguments(config=backend_config)
+        # Valide actor resources
+        self.actor_kwargs = self._validate_actor_arguments(config=backend_config)
+        self.pool: Optional[BasicActorPool] = None
 
-        self.pool = BasicActorPool(
-            actor_type=ClientAppActor,
-            client_resources=client_resources,
-            actor_kwargs=actor_kwargs,
-        )
+        self.app_fn: Optional[Callable[[], ClientApp]] = None
 
     def _validate_client_resources(self, config: BackendConfig) -> ClientResourcesDict:
         client_resources_config = config.get(self.client_resources_key)
@@ -120,20 +117,31 @@ class RayBackend(Backend):
     @property
     def num_workers(self) -> int:
         """Return number of actors in pool."""
-        return self.pool.num_actors
+        return self.pool.num_actors if self.pool else 0
 
     def is_worker_idle(self) -> bool:
         """Report whether the pool has idle actors."""
-        return self.pool.is_actor_available()
+        return self.pool.is_actor_available() if self.pool else False
 
-    def build(self) -> None:
+    def build(self, app_fn: Callable[[], ClientApp]) -> None:
         """Build pool of Ray actors that this backend will submit jobs to."""
+        # Create Actor Pool
+        try:
+            self.pool = BasicActorPool(
+                actor_type=ClientAppActor,
+                client_resources=self.client_resources,
+                actor_kwargs=self.actor_kwargs,
+            )
+        except Exception as ex:
+            raise ex
+
         self.pool.add_actors_to_pool(self.pool.actors_capacity)
+        # Set ClientApp callable that ray actors will use
+        self.app_fn = app_fn
         log(DEBUG, "Constructed ActorPool with: %i actors", self.pool.num_actors)
 
     def process_message(
         self,
-        app: Callable[[], ClientApp],
         message: Message,
         context: Context,
     ) -> Tuple[Message, Context]:
@@ -143,11 +151,20 @@ class RayBackend(Backend):
         """
         partition_id = context.node_config[PARTITION_ID_KEY]
 
+        if self.pool is None:
+            raise ValueError("The actor pool is empty, unfit to process messages.")
+
+        if self.app_fn is None:
+            raise ValueError(
+                "Unspecified function to load a `ClientApp`. "
+                "Call the backend's `build()` method before processing messages."
+            )
+
         try:
             # Submit a task to the pool
             future = self.pool.submit(
                 lambda a, a_fn, mssg, cid, state: a.run.remote(a_fn, mssg, cid, state),
-                (app, message, str(partition_id), context),
+                (self.app_fn, message, str(partition_id), context),
             )
 
             # Fetch result
@@ -170,6 +187,7 @@ class RayBackend(Backend):
 
     def terminate(self) -> None:
         """Terminate all actors in actor pool."""
-        self.pool.terminate_all_actors()
+        if self.pool:
+            self.pool.terminate_all_actors()
         ray.shutdown()
         log(DEBUG, "Terminated %s", self.__class__.__name__)

flwr/server/superlink/fleet/vce/vce_api.py

@@ -87,7 +87,6 @@ def _register_node_states(
 
 # pylint: disable=too-many-arguments,too-many-locals
 def worker(
-    app_fn: Callable[[], ClientApp],
     taskins_queue: "Queue[TaskIns]",
     taskres_queue: "Queue[TaskRes]",
     node_states: Dict[int, NodeState],
@@ -110,9 +109,7 @@ def worker(
             message = message_from_taskins(task_ins)
 
             # Let backend process message
-            out_mssg, updated_context = backend.process_message(
-                app_fn, message, context
-            )
+            out_mssg, updated_context = backend.process_message(message, context)
 
             # Update Context
             node_states[node_id].update_context(
@@ -193,7 +190,7 @@ def run_api(
         backend = backend_fn()
 
         # Build backend
-        backend.build()
+        backend.build(app_fn)
 
         # Add workers (they submit Messages to Backend)
         state = state_factory.state()
@@ -223,7 +220,6 @@ def run_api(
             _ = [
                 executor.submit(
                     worker,
-                    app_fn,
                     taskins_queue,
                     taskres_queue,
                     node_states,