flwr-nightly 1.11.0.dev20240822__py3-none-any.whl → 1.11.0.dev20240824__py3-none-any.whl

flwr/server/superlink/fleet/rest_rere/rest_api.py

@@ -15,17 +15,29 @@
 """Experimental REST API server."""
 
 
+from __future__ import annotations
+
 import sys
+from typing import Awaitable, Callable, TypeVar
+
+from google.protobuf.message import Message as GrpcMessage
 
 from flwr.common.constant import MISSING_EXTRA_REST
+from flwr.proto.fab_pb2 import GetFabRequest, GetFabResponse  # pylint: disable=E0611
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
+    CreateNodeResponse,
     DeleteNodeRequest,
+    DeleteNodeResponse,
     PingRequest,
+    PingResponse,
     PullTaskInsRequest,
+    PullTaskInsResponse,
     PushTaskResRequest,
+    PushTaskResResponse,
 )
-from flwr.proto.run_pb2 import GetRunRequest  # pylint: disable=E0611
+from flwr.proto.run_pb2 import GetRunRequest, GetRunResponse  # pylint: disable=E0611
+from flwr.server.superlink.ffs.ffs import Ffs
 from flwr.server.superlink.fleet.message_handler import message_handler
 from flwr.server.superlink.state import State
 
@@ -40,172 +52,108 @@ except ModuleNotFoundError:
     sys.exit(MISSING_EXTRA_REST)
 
 
-async def create_node(request: Request) -> Response:
-    """Create Node."""
-    _check_headers(request.headers)
+GrpcRequest = TypeVar("GrpcRequest", bound=GrpcMessage)
+GrpcResponse = TypeVar("GrpcResponse", bound=GrpcMessage)
 
-    # Get the request body as raw bytes
-    create_node_request_bytes: bytes = await request.body()
+GrpcAsyncFunction = Callable[[GrpcRequest], Awaitable[GrpcResponse]]
+RestEndPoint = Callable[[Request], Awaitable[Response]]
 
-    # Deserialize ProtoBuf
-    create_node_request_proto = CreateNodeRequest()
-    create_node_request_proto.ParseFromString(create_node_request_bytes)
 
-    # Get state from app
-    state: State = app.state.STATE_FACTORY.state()
+def rest_request_response(
+    grpc_request_type: type[GrpcRequest],
+) -> Callable[[GrpcAsyncFunction[GrpcRequest, GrpcResponse]], RestEndPoint]:
+    """Convert an async gRPC-based function into a RESTful HTTP endpoint."""
 
-    # Handle message
-    create_node_response_proto = message_handler.create_node(
-        request=create_node_request_proto, state=state
-    )
+    def decorator(func: GrpcAsyncFunction[GrpcRequest, GrpcResponse]) -> RestEndPoint:
+        async def wrapper(request: Request) -> Response:
+            _check_headers(request.headers)
 
-    # Return serialized ProtoBuf
-    create_node_response_bytes = create_node_response_proto.SerializeToString()
-    return Response(
-        status_code=200,
-        content=create_node_response_bytes,
-        headers={"Content-Type": "application/protobuf"},
-    )
+            # Get the request body as raw bytes
+            grpc_req_bytes: bytes = await request.body()
 
+            # Deserialize ProtoBuf
+            grpc_req = grpc_request_type.FromString(grpc_req_bytes)
+            grpc_res = await func(grpc_req)
+            return Response(
+                status_code=200,
+                content=grpc_res.SerializeToString(),
+                headers={"Content-Type": "application/protobuf"},
+            )
 
-async def delete_node(request: Request) -> Response:
-    """Delete Node Id."""
-    _check_headers(request.headers)
+        return wrapper
 
-    # Get the request body as raw bytes
-    delete_node_request_bytes: bytes = await request.body()
+    return decorator
 
-    # Deserialize ProtoBuf
-    delete_node_request_proto = DeleteNodeRequest()
-    delete_node_request_proto.ParseFromString(delete_node_request_bytes)
 
+@rest_request_response(CreateNodeRequest)
+async def create_node(request: CreateNodeRequest) -> CreateNodeResponse:
+    """Create Node."""
     # Get state from app
     state: State = app.state.STATE_FACTORY.state()
 
     # Handle message
-    delete_node_response_proto = message_handler.delete_node(
-        request=delete_node_request_proto, state=state
-    )
+    return message_handler.create_node(request=request, state=state)
 
-    # Return serialized ProtoBuf
-    delete_node_response_bytes = delete_node_response_proto.SerializeToString()
-    return Response(
-        status_code=200,
-        content=delete_node_response_bytes,
-        headers={"Content-Type": "application/protobuf"},
-    )
 
+@rest_request_response(DeleteNodeRequest)
+async def delete_node(request: DeleteNodeRequest) -> DeleteNodeResponse:
+    """Delete Node Id."""
+    # Get state from app
+    state: State = app.state.STATE_FACTORY.state()
 
-async def pull_task_ins(request: Request) -> Response:
-    """Pull TaskIns."""
-    _check_headers(request.headers)
-
-    # Get the request body as raw bytes
-    pull_task_ins_request_bytes: bytes = await request.body()
+    # Handle message
+    return message_handler.delete_node(request=request, state=state)
 
-    # Deserialize ProtoBuf
-    pull_task_ins_request_proto = PullTaskInsRequest()
-    pull_task_ins_request_proto.ParseFromString(pull_task_ins_request_bytes)
 
+@rest_request_response(PullTaskInsRequest)
+async def pull_task_ins(request: PullTaskInsRequest) -> PullTaskInsResponse:
+    """Pull TaskIns."""
     # Get state from app
     state: State = app.state.STATE_FACTORY.state()
 
     # Handle message
-    pull_task_ins_response_proto = message_handler.pull_task_ins(
-        request=pull_task_ins_request_proto,
-        state=state,
-    )
-
-    # Return serialized ProtoBuf
-    pull_task_ins_response_bytes = pull_task_ins_response_proto.SerializeToString()
-    return Response(
-        status_code=200,
-        content=pull_task_ins_response_bytes,
-        headers={"Content-Type": "application/protobuf"},
-    )
+    return message_handler.pull_task_ins(request=request, state=state)
 
 
-async def push_task_res(request: Request) -> Response:  # Check if token is needed here
+# Check if token is needed here
+@rest_request_response(PushTaskResRequest)
+async def push_task_res(request: PushTaskResRequest) -> PushTaskResResponse:
     """Push TaskRes."""
-    _check_headers(request.headers)
-
-    # Get the request body as raw bytes
-    push_task_res_request_bytes: bytes = await request.body()
-
-    # Deserialize ProtoBuf
-    push_task_res_request_proto = PushTaskResRequest()
-    push_task_res_request_proto.ParseFromString(push_task_res_request_bytes)
-
     # Get state from app
     state: State = app.state.STATE_FACTORY.state()
 
     # Handle message
-    push_task_res_response_proto = message_handler.push_task_res(
-        request=push_task_res_request_proto,
-        state=state,
-    )
-
-    # Return serialized ProtoBuf
-    push_task_res_response_bytes = push_task_res_response_proto.SerializeToString()
-    return Response(
-        status_code=200,
-        content=push_task_res_response_bytes,
-        headers={"Content-Type": "application/protobuf"},
-    )
+    return message_handler.push_task_res(request=request, state=state)
 
 
-async def ping(request: Request) -> Response:
+@rest_request_response(PingRequest)
+async def ping(request: PingRequest) -> PingResponse:
     """Ping."""
-    _check_headers(request.headers)
-
-    # Get the request body as raw bytes
-    ping_request_bytes: bytes = await request.body()
-
-    # Deserialize ProtoBuf
-    ping_request_proto = PingRequest()
-    ping_request_proto.ParseFromString(ping_request_bytes)
-
     # Get state from app
     state: State = app.state.STATE_FACTORY.state()
 
     # Handle message
-    ping_response_proto = message_handler.ping(request=ping_request_proto, state=state)
-
-    # Return serialized ProtoBuf
-    ping_response_bytes = ping_response_proto.SerializeToString()
-    return Response(
-        status_code=200,
-        content=ping_response_bytes,
-        headers={"Content-Type": "application/protobuf"},
-    )
+    return message_handler.ping(request=request, state=state)
 
 
-async def get_run(request: Request) -> Response:
+@rest_request_response(GetRunRequest)
+async def get_run(request: GetRunRequest) -> GetRunResponse:
     """GetRun."""
-    _check_headers(request.headers)
-
-    # Get the request body as raw bytes
-    get_run_request_bytes: bytes = await request.body()
-
-    # Deserialize ProtoBuf
-    get_run_request_proto = GetRunRequest()
-    get_run_request_proto.ParseFromString(get_run_request_bytes)
-
     # Get state from app
     state: State = app.state.STATE_FACTORY.state()
 
     # Handle message
-    get_run_response_proto = message_handler.get_run(
-        request=get_run_request_proto, state=state
-    )
+    return message_handler.get_run(request=request, state=state)
+
 
-    # Return serialized ProtoBuf
-    get_run_response_bytes = get_run_response_proto.SerializeToString()
-    return Response(
-        status_code=200,
-        content=get_run_response_bytes,
-        headers={"Content-Type": "application/protobuf"},
-    )
+@rest_request_response(GetFabRequest)
+async def get_fab(request: GetFabRequest) -> GetFabResponse:
+    """GetRun."""
+    # Get ffs from app
+    ffs: Ffs = app.state.FFS_FACTORY.state()
+
+    # Handle message
+    return message_handler.get_fab(request=request, ffs=ffs)
 
 
 routes = [
@@ -215,6 +163,7 @@ routes = [
     Route("/api/v0/fleet/push-task-res", push_task_res, methods=["POST"]),
     Route("/api/v0/fleet/ping", ping, methods=["POST"]),
     Route("/api/v0/fleet/get-run", get_run, methods=["POST"]),
+    Route("/api/v0/fleet/get-fab", get_fab, methods=["POST"]),
 ]
 
 app: Starlette = Starlette(

flwr/server/superlink/fleet/vce/backend/backend.py

@@ -33,7 +33,7 @@ class Backend(ABC):
         """Construct a backend."""
 
     @abstractmethod
-    def build(self) -> None:
+    def build(self, app_fn: Callable[[], ClientApp]) -> None:
         """Build backend.
 
         Different components need to be in place before workers in a backend are ready
@@ -60,7 +60,6 @@ class Backend(ABC):
     @abstractmethod
     def process_message(
         self,
-        app: Callable[[], ClientApp],
         message: Message,
         context: Context,
     ) -> Tuple[Message, Context]:

flwr/server/superlink/fleet/vce/backend/raybackend.py

@@ -16,7 +16,7 @@
 
 import sys
 from logging import DEBUG, ERROR
-from typing import Callable, Dict, Tuple, Union
+from typing import Callable, Dict, Optional, Tuple, Union
 
 import ray
 
@@ -63,6 +63,8 @@ class RayBackend(Backend):
             actor_kwargs=actor_kwargs,
         )
 
+        self.app_fn: Optional[Callable[[], ClientApp]] = None
+
     def _validate_client_resources(self, config: BackendConfig) -> ClientResourcesDict:
         client_resources_config = config.get(self.client_resources_key)
         client_resources: ClientResourcesDict = {}
@@ -126,14 +128,15 @@ class RayBackend(Backend):
         """Report whether the pool has idle actors."""
         return self.pool.is_actor_available()
 
-    def build(self) -> None:
+    def build(self, app_fn: Callable[[], ClientApp]) -> None:
         """Build pool of Ray actors that this backend will submit jobs to."""
         self.pool.add_actors_to_pool(self.pool.actors_capacity)
+        # Set ClientApp callable that ray actors will use
+        self.app_fn = app_fn
         log(DEBUG, "Constructed ActorPool with: %i actors", self.pool.num_actors)
 
     def process_message(
         self,
-        app: Callable[[], ClientApp],
         message: Message,
         context: Context,
     ) -> Tuple[Message, Context]:
@@ -143,11 +146,17 @@ class RayBackend(Backend):
         """
         partition_id = context.node_config[PARTITION_ID_KEY]
 
+        if self.app_fn is None:
+            raise ValueError(
+                "Unspecified function to load a `ClientApp`. "
+                "Call the backend's `build()` method before processing messages."
+            )
+
         try:
             # Submit a task to the pool
             future = self.pool.submit(
                 lambda a, a_fn, mssg, cid, state: a.run.remote(a_fn, mssg, cid, state),
-                (app, message, str(partition_id), context),
+                (self.app_fn, message, str(partition_id), context),
             )
 
             # Fetch result

flwr/server/superlink/fleet/vce/vce_api.py

@@ -87,7 +87,6 @@ def _register_node_states(
 
 # pylint: disable=too-many-arguments,too-many-locals
 def worker(
-    app_fn: Callable[[], ClientApp],
     taskins_queue: "Queue[TaskIns]",
     taskres_queue: "Queue[TaskRes]",
     node_states: Dict[int, NodeState],
@@ -110,9 +109,7 @@ def worker(
             message = message_from_taskins(task_ins)
 
             # Let backend process message
-            out_mssg, updated_context = backend.process_message(
-                app_fn, message, context
-            )
+            out_mssg, updated_context = backend.process_message(message, context)
 
             # Update Context
             node_states[node_id].update_context(
@@ -193,7 +190,7 @@ def run_api(
         backend = backend_fn()
 
         # Build backend
-        backend.build()
+        backend.build(app_fn)
 
         # Add workers (they submit Messages to Backend)
         state = state_factory.state()
@@ -223,7 +220,6 @@ def run_api(
             _ = [
                 executor.submit(
                     worker,
-                    app_fn,
                     taskins_queue,
                     taskres_queue,
                     node_states,

flwr/server/superlink/state/in_memory_state.py

@@ -45,7 +45,7 @@ class InMemoryState(State):  # pylint: disable=R0902,R0904
         self.task_ins_store: Dict[UUID, TaskIns] = {}
         self.task_res_store: Dict[UUID, TaskRes] = {}
 
-        self.client_public_keys: Set[bytes] = set()
+        self.node_public_keys: Set[bytes] = set()
         self.server_public_key: Optional[bytes] = None
         self.server_private_key: Optional[bytes] = None
 
@@ -237,7 +237,7 @@ class InMemoryState(State):  # pylint: disable=R0902,R0904
             return node_id
 
     def delete_node(self, node_id: int, public_key: Optional[bytes] = None) -> None:
-        """Delete a client node."""
+        """Delete a node."""
         with self.lock:
             if node_id not in self.node_ids:
                 raise ValueError(f"Node {node_id} not found")
@@ -254,7 +254,7 @@ class InMemoryState(State):  # pylint: disable=R0902,R0904
             del self.node_ids[node_id]
 
     def get_nodes(self, run_id: int) -> Set[int]:
-        """Return all available client nodes.
+        """Return all available nodes.
 
         Constraints
         -----------
@@ -271,9 +271,9 @@ class InMemoryState(State):  # pylint: disable=R0902,R0904
                 if online_until > current_time
             }
 
-    def get_node_id(self, client_public_key: bytes) -> Optional[int]:
-        """Retrieve stored `node_id` filtered by `client_public_keys`."""
-        return self.public_key_to_node_id.get(client_public_key)
+    def get_node_id(self, node_public_key: bytes) -> Optional[int]:
+        """Retrieve stored `node_id` filtered by `node_public_keys`."""
+        return self.public_key_to_node_id.get(node_public_key)
 
     def create_run(
         self,
@@ -318,19 +318,19 @@ class InMemoryState(State):  # pylint: disable=R0902,R0904
         """Retrieve `server_public_key` in urlsafe bytes."""
         return self.server_public_key
 
-    def store_client_public_keys(self, public_keys: Set[bytes]) -> None:
-        """Store a set of `client_public_keys` in state."""
+    def store_node_public_keys(self, public_keys: Set[bytes]) -> None:
+        """Store a set of `node_public_keys` in state."""
         with self.lock:
-            self.client_public_keys = public_keys
+            self.node_public_keys = public_keys
 
-    def store_client_public_key(self, public_key: bytes) -> None:
-        """Store a `client_public_key` in state."""
+    def store_node_public_key(self, public_key: bytes) -> None:
+        """Store a `node_public_key` in state."""
         with self.lock:
-            self.client_public_keys.add(public_key)
+            self.node_public_keys.add(public_key)
 
-    def get_client_public_keys(self) -> Set[bytes]:
-        """Retrieve all currently stored `client_public_keys` as a set."""
-        return self.client_public_keys
+    def get_node_public_keys(self) -> Set[bytes]:
+        """Retrieve all currently stored `node_public_keys` as a set."""
+        return self.node_public_keys
 
     def get_run(self, run_id: int) -> Optional[Run]:
         """Retrieve information about the run with the specified `run_id`."""

flwr/server/superlink/state/sqlite_state.py

@@ -569,7 +569,7 @@ class SqliteState(State):  # pylint: disable=R0904
         return node_id
 
     def delete_node(self, node_id: int, public_key: Optional[bytes] = None) -> None:
-        """Delete a client node."""
+        """Delete a node."""
         query = "DELETE FROM node WHERE node_id = ?"
         params = (node_id,)
 
@@ -607,10 +607,10 @@ class SqliteState(State):  # pylint: disable=R0904
         result: Set[int] = {row["node_id"] for row in rows}
         return result
 
-    def get_node_id(self, client_public_key: bytes) -> Optional[int]:
-        """Retrieve stored `node_id` filtered by `client_public_keys`."""
+    def get_node_id(self, node_public_key: bytes) -> Optional[int]:
+        """Retrieve stored `node_id` filtered by `node_public_keys`."""
         query = "SELECT node_id FROM node WHERE public_key = :public_key;"
-        row = self.query(query, {"public_key": client_public_key})
+        row = self.query(query, {"public_key": node_public_key})
         if len(row) > 0:
             node_id: int = row[0]["node_id"]
             return node_id
@@ -684,19 +684,19 @@ class SqliteState(State):  # pylint: disable=R0904
             public_key = None
         return public_key
 
-    def store_client_public_keys(self, public_keys: Set[bytes]) -> None:
-        """Store a set of `client_public_keys` in state."""
+    def store_node_public_keys(self, public_keys: Set[bytes]) -> None:
+        """Store a set of `node_public_keys` in state."""
         query = "INSERT INTO public_key (public_key) VALUES (?)"
         data = [(key,) for key in public_keys]
         self.query(query, data)
 
-    def store_client_public_key(self, public_key: bytes) -> None:
-        """Store a `client_public_key` in state."""
+    def store_node_public_key(self, public_key: bytes) -> None:
+        """Store a `node_public_key` in state."""
         query = "INSERT INTO public_key (public_key) VALUES (:public_key)"
         self.query(query, {"public_key": public_key})
 
-    def get_client_public_keys(self) -> Set[bytes]:
-        """Retrieve all currently stored `client_public_keys` as a set."""
+    def get_node_public_keys(self) -> Set[bytes]:
+        """Retrieve all currently stored `node_public_keys` as a set."""
         query = "SELECT public_key FROM public_key"
         rows = self.query(query)
         result: Set[bytes] = {row["public_key"] for row in rows}

flwr/server/superlink/state/state.py

@@ -153,8 +153,8 @@ class State(abc.ABC):  # pylint: disable=R0904
         """
 
     @abc.abstractmethod
-    def get_node_id(self, client_public_key: bytes) -> Optional[int]:
-        """Retrieve stored `node_id` filtered by `client_public_keys`."""
+    def get_node_id(self, node_public_key: bytes) -> Optional[int]:
+        """Retrieve stored `node_id` filtered by `node_public_keys`."""
 
     @abc.abstractmethod
     def create_run(
@@ -199,16 +199,16 @@ class State(abc.ABC):  # pylint: disable=R0904
         """Retrieve `server_public_key` in urlsafe bytes."""
 
     @abc.abstractmethod
-    def store_client_public_keys(self, public_keys: Set[bytes]) -> None:
-        """Store a set of `client_public_keys` in state."""
+    def store_node_public_keys(self, public_keys: Set[bytes]) -> None:
+        """Store a set of `node_public_keys` in state."""
 
     @abc.abstractmethod
-    def store_client_public_key(self, public_key: bytes) -> None:
-        """Store a `client_public_key` in state."""
+    def store_node_public_key(self, public_key: bytes) -> None:
+        """Store a `node_public_key` in state."""
 
     @abc.abstractmethod
-    def get_client_public_keys(self) -> Set[bytes]:
-        """Retrieve all currently stored `client_public_keys` as a set."""
+    def get_node_public_keys(self) -> Set[bytes]:
+        """Retrieve all currently stored `node_public_keys` as a set."""
 
     @abc.abstractmethod
     def acknowledge_ping(self, node_id: int, ping_interval: float) -> bool:

flwr/server/workflow/secure_aggregation/secagg_workflow.py

@@ -35,6 +35,7 @@ class SecAggWorkflow(SecAggPlusWorkflow):
     contributions to compute the weighted average of model parameters.
 
     The protocol involves four main stages:
+
     - 'setup': Send SecAgg configuration to clients and collect their public keys.
     - 'share keys': Broadcast public keys among clients and collect encrypted secret
       key shares.

flwr/server/workflow/secure_aggregation/secaggplus_workflow.py

@@ -99,6 +99,7 @@ class SecAggPlusWorkflow:
     contributions to compute the weighted average of model parameters.
 
     The protocol involves four main stages:
+
     - 'setup': Send SecAgg+ configuration to clients and collect their public keys.
     - 'share keys': Broadcast public keys among clients and collect encrypted secret
       key shares.