flwr-nightly 1.8.0.dev20240315__py3-none-any.whl → 1.11.0.dev20240813__py3-none-any.whl

flwr/server/superlink/driver/driver_grpc.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Flower Labs GmbH. All Rights Reserved.
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

flwr/server/superlink/driver/driver_servicer.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Flower Labs GmbH. All Rights Reserved.
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,13 +15,15 @@
 """Driver API servicer."""
 
 
-from logging import DEBUG, INFO
+import time
+from logging import DEBUG
 from typing import List, Optional, Set
 from uuid import UUID
 
 import grpc
 
 from flwr.common.logger import log
+from flwr.common.serde import user_config_from_proto, user_config_to_proto
 from flwr.proto import driver_pb2_grpc  # pylint: disable=E0611
 from flwr.proto.driver_pb2 import (  # pylint: disable=E0611
     CreateRunRequest,
@@ -33,7 +35,13 @@ from flwr.proto.driver_pb2 import (  # pylint: disable=E0611
     PushTaskInsRequest,
     PushTaskInsResponse,
 )
+from flwr.proto.fab_pb2 import GetFabRequest, GetFabResponse  # pylint: disable=E0611
 from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
+from flwr.proto.run_pb2 import (  # pylint: disable=E0611
+    GetRunRequest,
+    GetRunResponse,
+    Run,
+)
 from flwr.proto.task_pb2 import TaskRes  # pylint: disable=E0611
 from flwr.server.superlink.state import State, StateFactory
 from flwr.server.utils.validator import validate_task_ins_or_res
@@ -61,9 +69,13 @@ class DriverServicer(driver_pb2_grpc.DriverServicer):
         self, request: CreateRunRequest, context: grpc.ServicerContext
     ) -> CreateRunResponse:
         """Create run ID."""
-        log(INFO, "DriverServicer.CreateRun")
+        log(DEBUG, "DriverServicer.CreateRun")
         state: State = self.state_factory.state()
-        run_id = state.create_run()
+        run_id = state.create_run(
+            request.fab_id,
+            request.fab_version,
+            user_config_from_proto(request.override_config),
+        )
         return CreateRunResponse(run_id=run_id)
 
     def PushTaskIns(
@@ -72,6 +84,11 @@ class DriverServicer(driver_pb2_grpc.DriverServicer):
         """Push a set of TaskIns."""
         log(DEBUG, "DriverServicer.PushTaskIns")
 
+        # Set pushed_at (timestamp in seconds)
+        pushed_at = time.time()
+        for task_ins in request.task_ins_list:
+            task_ins.task.pushed_at = pushed_at
+
         # Validate request
         _raise_if(len(request.task_ins_list) == 0, "`task_ins_list` must not be empty")
         for task_ins in request.task_ins_list:
@@ -123,6 +140,36 @@ class DriverServicer(driver_pb2_grpc.DriverServicer):
         context.set_code(grpc.StatusCode.OK)
         return PullTaskResResponse(task_res_list=task_res_list)
 
+    def GetRun(
+        self, request: GetRunRequest, context: grpc.ServicerContext
+    ) -> GetRunResponse:
+        """Get run information."""
+        log(DEBUG, "DriverServicer.GetRun")
+
+        # Init state
+        state: State = self.state_factory.state()
+
+        # Retrieve run information
+        run = state.get_run(request.run_id)
+
+        if run is None:
+            return GetRunResponse()
+
+        return GetRunResponse(
+            run=Run(
+                run_id=run.run_id,
+                fab_id=run.fab_id,
+                fab_version=run.fab_version,
+                override_config=user_config_to_proto(run.override_config),
+            )
+        )
+
+    def GetFab(
+        self, request: GetFabRequest, context: grpc.ServicerContext
+    ) -> GetFabResponse:
+        """Will be implemented later."""
+        raise NotImplementedError
+
 
 def _raise_if(validation_error: bool, detail: str) -> None:
     if validation_error:

flwr/server/superlink/ffs/__init__.py

@@ -0,0 +1,24 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower File Storage for large objects."""
+
+
+from .disk_ffs import DiskFfs as DiskFfs
+from .ffs import Ffs as Ffs
+
+__all__ = [
+    "DiskFfs",
+    "Ffs",
+]

flwr/server/superlink/ffs/disk_ffs.py

@@ -0,0 +1,104 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Disk based Flower File Storage."""
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Dict, List, Tuple
+
+from flwr.server.superlink.ffs.ffs import Ffs
+
+
+class DiskFfs(Ffs):  # pylint: disable=R0904
+    """Disk-based Flower File Storage interface for large objects."""
+
+    def __init__(self, base_dir: str) -> None:
+        """Create a new DiskFfs instance.
+
+        Parameters
+        ----------
+        base_dir : str
+            The base directory to store the objects.
+        """
+        self.base_dir = Path(base_dir)
+
+    def put(self, content: bytes, meta: Dict[str, str]) -> str:
+        """Store bytes and metadata and return key (hash of content).
+
+        Parameters
+        ----------
+        content : bytes
+            The content to be stored.
+        meta : Dict[str, str]
+            The metadata to be stored.
+
+        Returns
+        -------
+        key : str
+            The key (sha256hex hash) of the content.
+        """
+        content_hash = hashlib.sha256(content).hexdigest()
+
+        self.base_dir.mkdir(exist_ok=True, parents=True)
+        (self.base_dir / content_hash).write_bytes(content)
+        (self.base_dir / f"{content_hash}.META").write_text(json.dumps(meta))
+
+        return content_hash
+
+    def get(self, key: str) -> Tuple[bytes, Dict[str, str]]:
+        """Return tuple containing the object content and metadata.
+
+        Parameters
+        ----------
+        key : str
+            The sha256hex hash of the object to be retrieved.
+
+        Returns
+        -------
+        Tuple[bytes, Dict[str, str]]
+            A tuple containing the object content and metadata.
+        """
+        content = (self.base_dir / key).read_bytes()
+        meta = json.loads((self.base_dir / f"{key}.META").read_text())
+
+        return content, meta
+
+    def delete(self, key: str) -> None:
+        """Delete object with hash.
+
+        Parameters
+        ----------
+        key : str
+            The sha256hex hash of the object to be deleted.
+        """
+        (self.base_dir / key).unlink()
+        (self.base_dir / f"{key}.META").unlink()
+
+    def list(self) -> List[str]:
+        """List all keys.
+
+        Return all available keys in this `Ffs` instance.
+        This can be combined with, for example,
+        the `delete` method to delete objects.
+
+        Returns
+        -------
+        List[str]
+            A list of all available keys.
+        """
+        return [
+            item.name for item in self.base_dir.iterdir() if not item.suffix == ".META"
+        ]

flwr/server/superlink/ffs/ffs.py

@@ -0,0 +1,79 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Abstract base class for Flower File Storage interface."""
+
+
+import abc
+from typing import Dict, List, Tuple
+
+
+class Ffs(abc.ABC):  # pylint: disable=R0904
+    """Abstract Flower File Storage interface for large objects."""
+
+    @abc.abstractmethod
+    def put(self, content: bytes, meta: Dict[str, str]) -> str:
+        """Store bytes and metadata and return sha256hex hash of data as str.
+
+        Parameters
+        ----------
+        content : bytes
+            The content to be stored.
+        meta : Dict[str, str]
+            The metadata to be stored.
+
+        Returns
+        -------
+        key : str
+            The key (sha256hex hash) of the content.
+        """
+
+    @abc.abstractmethod
+    def get(self, key: str) -> Tuple[bytes, Dict[str, str]]:
+        """Return tuple containing the object content and metadata.
+
+        Parameters
+        ----------
+        key : str
+            The key (sha256hex hash) of the object to be retrieved.
+
+        Returns
+        -------
+        Tuple[bytes, Dict[str, str]]
+            A tuple containing the object content and metadata.
+        """
+
+    @abc.abstractmethod
+    def delete(self, key: str) -> None:
+        """Delete object with hash.
+
+        Parameters
+        ----------
+        key : str
+            The key (sha256hex hash) of the object to be deleted.
+        """
+
+    @abc.abstractmethod
+    def list(self) -> List[str]:
+        """List keys of all stored objects.
+
+        Return all available keys in this `Ffs` instance.
+        This can be combined with, for example,
+        the `delete` method to delete objects.
+
+        Returns
+        -------
+        List[str]
+            A list of all available keys.
+        """

flwr/server/superlink/fleet/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Flower Labs GmbH. All Rights Reserved.
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

flwr/server/superlink/fleet/grpc_adapter/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Server-side part of the GrpcAdapter transport layer."""

flwr/server/superlink/fleet/grpc_adapter/grpc_adapter_servicer.py

@@ -0,0 +1,131 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Fleet API gRPC adapter servicer."""
+
+
+from logging import DEBUG, INFO
+from typing import Callable, Type, TypeVar
+
+import grpc
+from google.protobuf.message import Message as GrpcMessage
+
+from flwr.common.logger import log
+from flwr.proto import grpcadapter_pb2_grpc  # pylint: disable=E0611
+from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
+    CreateNodeRequest,
+    CreateNodeResponse,
+    DeleteNodeRequest,
+    DeleteNodeResponse,
+    PingRequest,
+    PingResponse,
+    PullTaskInsRequest,
+    PullTaskInsResponse,
+    PushTaskResRequest,
+    PushTaskResResponse,
+)
+from flwr.proto.grpcadapter_pb2 import MessageContainer  # pylint: disable=E0611
+from flwr.proto.run_pb2 import GetRunRequest, GetRunResponse  # pylint: disable=E0611
+from flwr.server.superlink.fleet.message_handler import message_handler
+from flwr.server.superlink.state import StateFactory
+
+T = TypeVar("T", bound=GrpcMessage)
+
+
+def _handle(
+    msg_container: MessageContainer,
+    request_type: Type[T],
+    handler: Callable[[T], GrpcMessage],
+) -> MessageContainer:
+    req = request_type.FromString(msg_container.grpc_message_content)
+    res = handler(req)
+    return MessageContainer(
+        metadata={},
+        grpc_message_name=res.__class__.__qualname__,
+        grpc_message_content=res.SerializeToString(),
+    )
+
+
+class GrpcAdapterServicer(grpcadapter_pb2_grpc.GrpcAdapterServicer):
+    """Fleet API via GrpcAdapter servicer."""
+
+    def __init__(self, state_factory: StateFactory) -> None:
+        self.state_factory = state_factory
+
+    def SendReceive(
+        self, request: MessageContainer, context: grpc.ServicerContext
+    ) -> MessageContainer:
+        """."""
+        log(DEBUG, "GrpcAdapterServicer.SendReceive")
+        if request.grpc_message_name == CreateNodeRequest.__qualname__:
+            return _handle(request, CreateNodeRequest, self._create_node)
+        if request.grpc_message_name == DeleteNodeRequest.__qualname__:
+            return _handle(request, DeleteNodeRequest, self._delete_node)
+        if request.grpc_message_name == PingRequest.__qualname__:
+            return _handle(request, PingRequest, self._ping)
+        if request.grpc_message_name == PullTaskInsRequest.__qualname__:
+            return _handle(request, PullTaskInsRequest, self._pull_task_ins)
+        if request.grpc_message_name == PushTaskResRequest.__qualname__:
+            return _handle(request, PushTaskResRequest, self._push_task_res)
+        if request.grpc_message_name == GetRunRequest.__qualname__:
+            return _handle(request, GetRunRequest, self._get_run)
+        raise ValueError(f"Invalid grpc_message_name: {request.grpc_message_name}")
+
+    def _create_node(self, request: CreateNodeRequest) -> CreateNodeResponse:
+        """."""
+        log(INFO, "GrpcAdapter.CreateNode")
+        return message_handler.create_node(
+            request=request,
+            state=self.state_factory.state(),
+        )
+
+    def _delete_node(self, request: DeleteNodeRequest) -> DeleteNodeResponse:
+        """."""
+        log(INFO, "GrpcAdapter.DeleteNode")
+        return message_handler.delete_node(
+            request=request,
+            state=self.state_factory.state(),
+        )
+
+    def _ping(self, request: PingRequest) -> PingResponse:
+        """."""
+        log(DEBUG, "GrpcAdapter.Ping")
+        return message_handler.ping(
+            request=request,
+            state=self.state_factory.state(),
+        )
+
+    def _pull_task_ins(self, request: PullTaskInsRequest) -> PullTaskInsResponse:
+        """Pull TaskIns."""
+        log(INFO, "GrpcAdapter.PullTaskIns")
+        return message_handler.pull_task_ins(
+            request=request,
+            state=self.state_factory.state(),
+        )
+
+    def _push_task_res(self, request: PushTaskResRequest) -> PushTaskResResponse:
+        """Push TaskRes."""
+        log(INFO, "GrpcAdapter.PushTaskRes")
+        return message_handler.push_task_res(
+            request=request,
+            state=self.state_factory.state(),
+        )
+
+    def _get_run(self, request: GetRunRequest) -> GetRunResponse:
+        """Get run information."""
+        log(INFO, "GrpcAdapter.GetRun")
+        return message_handler.get_run(
+            request=request,
+            state=self.state_factory.state(),
+        )

flwr/server/superlink/fleet/grpc_bidi/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Flower Labs GmbH. All Rights Reserved.
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

flwr/server/superlink/fleet/grpc_bidi/flower_service_servicer.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Flower Labs GmbH. All Rights Reserved.
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

flwr/server/superlink/fleet/grpc_bidi/grpc_bridge.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Flower Labs GmbH. All Rights Reserved.
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

flwr/server/superlink/fleet/grpc_bidi/grpc_client_proxy.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Flower Labs GmbH. All Rights Reserved.
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

flwr/server/superlink/fleet/grpc_bidi/grpc_server.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Flower Labs GmbH. All Rights Reserved.
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@
 import concurrent.futures
 import sys
 from logging import ERROR
-from typing import Any, Callable, Optional, Tuple, Union
+from typing import Any, Callable, Optional, Sequence, Tuple, Union
 
 import grpc
 
@@ -29,6 +29,9 @@ from flwr.proto.transport_pb2_grpc import (  # pylint: disable=E0611
 )
 from flwr.server.client_manager import ClientManager
 from flwr.server.superlink.driver.driver_servicer import DriverServicer
+from flwr.server.superlink.fleet.grpc_adapter.grpc_adapter_servicer import (
+    GrpcAdapterServicer,
+)
 from flwr.server.superlink.fleet.grpc_bidi.flower_service_servicer import (
     FlowerServiceServicer,
 )
@@ -154,6 +157,7 @@ def start_grpc_server(  # pylint: disable=too-many-arguments
 def generic_create_grpc_server(  # pylint: disable=too-many-arguments
     servicer_and_add_fn: Union[
         Tuple[FleetServicer, AddServicerToServerFn],
+        Tuple[GrpcAdapterServicer, AddServicerToServerFn],
         Tuple[FlowerServiceServicer, AddServicerToServerFn],
         Tuple[DriverServicer, AddServicerToServerFn],
     ],
@@ -162,6 +166,7 @@ def generic_create_grpc_server(  # pylint: disable=too-many-arguments
     max_message_length: int = GRPC_MAX_MESSAGE_LENGTH,
     keepalive_time_ms: int = 210000,
     certificates: Optional[Tuple[bytes, bytes, bytes]] = None,
+    interceptors: Optional[Sequence[grpc.ServerInterceptor]] = None,
 ) -> grpc.Server:
     """Create a gRPC server with a single servicer.
 
@@ -249,6 +254,7 @@ def generic_create_grpc_server(  # pylint: disable=too-many-arguments
         # returning RESOURCE_EXHAUSTED status, or None to indicate no limit.
         maximum_concurrent_rpcs=max_concurrent_workers,
         options=options,
+        interceptors=interceptors,
     )
     add_servicer_to_server_fn(servicer, server)
 

flwr/server/superlink/fleet/grpc_rere/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Flower Labs GmbH. All Rights Reserved.
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

flwr/server/superlink/fleet/grpc_rere/fleet_servicer.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Flower Labs GmbH. All Rights Reserved.
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,22 +15,26 @@
 """Fleet API gRPC request-response servicer."""
 
 
-from logging import INFO
+from logging import DEBUG, INFO
 
 import grpc
 
 from flwr.common.logger import log
 from flwr.proto import fleet_pb2_grpc  # pylint: disable=E0611
+from flwr.proto.fab_pb2 import GetFabRequest, GetFabResponse  # pylint: disable=E0611
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     CreateNodeResponse,
     DeleteNodeRequest,
     DeleteNodeResponse,
+    PingRequest,
+    PingResponse,
     PullTaskInsRequest,
     PullTaskInsResponse,
     PushTaskResRequest,
     PushTaskResResponse,
 )
+from flwr.proto.run_pb2 import GetRunRequest, GetRunResponse  # pylint: disable=E0611
 from flwr.server.superlink.fleet.message_handler import message_handler
 from flwr.server.superlink.state import StateFactory
 
@@ -61,6 +65,14 @@ class FleetServicer(fleet_pb2_grpc.FleetServicer):
             state=self.state_factory.state(),
         )
 
+    def Ping(self, request: PingRequest, context: grpc.ServicerContext) -> PingResponse:
+        """."""
+        log(DEBUG, "FleetServicer.Ping")
+        return message_handler.ping(
+            request=request,
+            state=self.state_factory.state(),
+        )
+
     def PullTaskIns(
         self, request: PullTaskInsRequest, context: grpc.ServicerContext
     ) -> PullTaskInsResponse:
@@ -80,3 +92,19 @@ class FleetServicer(fleet_pb2_grpc.FleetServicer):
             request=request,
             state=self.state_factory.state(),
         )
+
+    def GetRun(
+        self, request: GetRunRequest, context: grpc.ServicerContext
+    ) -> GetRunResponse:
+        """Get run information."""
+        log(INFO, "FleetServicer.GetRun")
+        return message_handler.get_run(
+            request=request,
+            state=self.state_factory.state(),
+        )
+
+    def GetFab(
+        self, request: GetFabRequest, context: grpc.ServicerContext
+    ) -> GetFabResponse:
+        """Will be implemented later."""
+        raise NotImplementedError