flwr-nightly 1.10.0.dev20240612__py3-none-any.whl → 1.10.0.dev20240619__py3-none-any.whl

flwr/client/grpc_rere_client/grpc_adapter.py

@@ -0,0 +1,133 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""GrpcAdapter implementation."""
+
+
+import sys
+from logging import DEBUG
+from typing import Any, Type, TypeVar, cast
+
+import grpc
+from google.protobuf.message import Message as GrpcMessage
+
+from flwr.common import log
+from flwr.common.constant import (
+    GRPC_ADAPTER_METADATA_FLOWER_VERSION_KEY,
+    GRPC_ADAPTER_METADATA_SHOULD_EXIT_KEY,
+)
+from flwr.common.version import package_version
+from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
+    CreateNodeRequest,
+    CreateNodeResponse,
+    DeleteNodeRequest,
+    DeleteNodeResponse,
+    PingRequest,
+    PingResponse,
+    PullTaskInsRequest,
+    PullTaskInsResponse,
+    PushTaskResRequest,
+    PushTaskResResponse,
+)
+from flwr.proto.grpcadapter_pb2 import MessageContainer  # pylint: disable=E0611
+from flwr.proto.grpcadapter_pb2_grpc import GrpcAdapterStub
+from flwr.proto.run_pb2 import GetRunRequest, GetRunResponse  # pylint: disable=E0611
+
+T = TypeVar("T", bound=GrpcMessage)
+
+
+class GrpcAdapter:
+    """Adapter class to send and receive gRPC messages via the ``GrpcAdapterStub``.
+
+    This class utilizes the ``GrpcAdapterStub`` to send and receive gRPC messages
+    which are defined and used by the Fleet API, as defined in ``fleet.proto``.
+    """
+
+    def __init__(self, channel: grpc.Channel) -> None:
+        self.stub = GrpcAdapterStub(channel)
+
+    def _send_and_receive(
+        self, request: GrpcMessage, response_type: Type[T], **kwargs: Any
+    ) -> T:
+        # Serialize request
+        container_req = MessageContainer(
+            metadata={GRPC_ADAPTER_METADATA_FLOWER_VERSION_KEY: package_version},
+            grpc_message_name=request.__class__.__qualname__,
+            grpc_message_content=request.SerializeToString(),
+        )
+
+        # Send via the stub
+        container_res = cast(
+            MessageContainer, self.stub.SendReceive(container_req, **kwargs)
+        )
+
+        # Handle control message
+        should_exit = (
+            container_res.metadata.get(GRPC_ADAPTER_METADATA_SHOULD_EXIT_KEY, "false")
+            == "true"
+        )
+        if should_exit:
+            log(
+                DEBUG,
+                'Received shutdown signal: exit flag is set to ``"true"``. Exiting...',
+            )
+            sys.exit(0)
+
+        # Check the grpc_message_name of the response
+        if container_res.grpc_message_name != response_type.__qualname__:
+            raise ValueError(
+                f"Invalid grpc_message_name. Expected {response_type.__qualname__}"
+                f", but got {container_res.grpc_message_name}."
+            )
+
+        # Deserialize response
+        response = response_type()
+        response.ParseFromString(container_res.grpc_message_content)
+        return response
+
+    def CreateNode(  # pylint: disable=C0103
+        self, request: CreateNodeRequest, **kwargs: Any
+    ) -> CreateNodeResponse:
+        """."""
+        return self._send_and_receive(request, CreateNodeResponse, **kwargs)
+
+    def DeleteNode(  # pylint: disable=C0103
+        self, request: DeleteNodeRequest, **kwargs: Any
+    ) -> DeleteNodeResponse:
+        """."""
+        return self._send_and_receive(request, DeleteNodeResponse, **kwargs)
+
+    def Ping(  # pylint: disable=C0103
+        self, request: PingRequest, **kwargs: Any
+    ) -> PingResponse:
+        """."""
+        return self._send_and_receive(request, PingResponse, **kwargs)
+
+    def PullTaskIns(  # pylint: disable=C0103
+        self, request: PullTaskInsRequest, **kwargs: Any
+    ) -> PullTaskInsResponse:
+        """."""
+        return self._send_and_receive(request, PullTaskInsResponse, **kwargs)
+
+    def PushTaskRes(  # pylint: disable=C0103
+        self, request: PushTaskResRequest, **kwargs: Any
+    ) -> PushTaskResResponse:
+        """."""
+        return self._send_and_receive(request, PushTaskResResponse, **kwargs)
+
+    def GetRun(  # pylint: disable=C0103
+        self, request: GetRunRequest, **kwargs: Any
+    ) -> GetRunResponse:
+        """."""
+        return self._send_and_receive(request, GetRunResponse, **kwargs)

flwr/client/mod/__init__.py

@@ -22,12 +22,12 @@ from .secure_aggregation import secagg_mod, secaggplus_mod
 from .utils import make_ffn
 
 __all__ = [
+    "LocalDpMod",
     "adaptiveclipping_mod",
     "fixedclipping_mod",
-    "LocalDpMod",
     "make_ffn",
-    "secagg_mod",
-    "secaggplus_mod",
     "message_size_mod",
     "parameters_size_mod",
+    "secagg_mod",
+    "secaggplus_mod",
 ]

flwr/client/rest_client/connection.py

@@ -117,10 +117,16 @@ def http_request_response(  # pylint: disable=,R0913, R0914, R0915
         Path of the root certificate. If provided, a secure
         connection using the certificates will be established to an SSL-enabled
         Flower server. Bytes won't work for the REST API.
+    authentication_keys : Optional[Tuple[PrivateKey, PublicKey]] (default: None)
+        Client authentication is not supported for this transport type.
 
     Returns
     -------
-    receive, send : Callable, Callable
+    receive : Callable
+    send : Callable
+    create_node : Optional[Callable]
+    delete_node : Optional[Callable]
+    get_run : Optional[Callable]
     """
     log(
         WARN,
@@ -145,6 +151,8 @@ def http_request_response(  # pylint: disable=,R0913, R0914, R0915
             "For the REST API, the root certificates "
             "must be provided as a string path to the client.",
         )
+    if authentication_keys is not None:
+        log(ERROR, "Client authentication is not supported for this transport type.")
 
     # Shared variables for inner functions
     metadata: Optional[Metadata] = None

flwr/client/supernode/app.py

@@ -29,6 +29,12 @@ from cryptography.hazmat.primitives.serialization import (
 
 from flwr.client.client_app import ClientApp, LoadClientAppError
 from flwr.common import EventType, event
+from flwr.common.config import get_flwr_dir, get_project_config, get_project_dir
+from flwr.common.constant import (
+    TRANSPORT_TYPE_GRPC_ADAPTER,
+    TRANSPORT_TYPE_GRPC_RERE,
+    TRANSPORT_TYPE_REST,
+)
 from flwr.common.exit_handlers import register_exit_handlers
 from flwr.common.logger import log, warn_deprecated_feature
 from flwr.common.object_ref import load_app, validate
@@ -44,11 +50,23 @@ def run_supernode() -> None:
 
     event(EventType.RUN_SUPERNODE_ENTER)
 
-    _ = _parse_args_run_supernode().parse_args()
+    args = _parse_args_run_supernode().parse_args()
+
+    _warn_deprecated_server_arg(args)
+
+    root_certificates = _get_certificates(args)
+    load_fn = _get_load_client_app_fn(args, multi_app=True)
+    authentication_keys = _try_setup_client_authentication(args)
 
-    log(
-        DEBUG,
-        "Flower SuperNode starting...",
+    _start_client_internal(
+        server_address=args.superlink,
+        load_client_app_fn=load_fn,
+        transport=args.transport,
+        root_certificates=root_certificates,
+        insecure=args.insecure,
+        authentication_keys=authentication_keys,
+        max_retries=args.max_retries,
+        max_wait_time=args.max_wait_time,
     )
 
     # Graceful shutdown
@@ -65,6 +83,27 @@ def run_client_app() -> None:
 
     args = _parse_args_run_client_app().parse_args()
 
+    _warn_deprecated_server_arg(args)
+
+    root_certificates = _get_certificates(args)
+    load_fn = _get_load_client_app_fn(args, multi_app=False)
+    authentication_keys = _try_setup_client_authentication(args)
+
+    _start_client_internal(
+        server_address=args.superlink,
+        load_client_app_fn=load_fn,
+        transport=args.transport,
+        root_certificates=root_certificates,
+        insecure=args.insecure,
+        authentication_keys=authentication_keys,
+        max_retries=args.max_retries,
+        max_wait_time=args.max_wait_time,
+    )
+    register_exit_handlers(event_type=EventType.RUN_CLIENT_APP_LEAVE)
+
+
+def _warn_deprecated_server_arg(args: argparse.Namespace) -> None:
+    """Warn about the deprecated argument `--server`."""
     if args.server != ADDRESS_FLEET_API_GRPC_RERE:
         warn = "Passing flag --server is deprecated. Use --superlink instead."
         warn_deprecated_feature(warn)
@@ -82,27 +121,6 @@ def run_client_app() -> None:
         else:
             args.superlink = args.server
 
-    root_certificates = _get_certificates(args)
-    log(
-        DEBUG,
-        "Flower will load ClientApp `%s`",
-        getattr(args, "client-app"),
-    )
-    load_fn = _get_load_client_app_fn(args)
-    authentication_keys = _try_setup_client_authentication(args)
-
-    _start_client_internal(
-        server_address=args.superlink,
-        load_client_app_fn=load_fn,
-        transport="rest" if args.rest else "grpc-rere",
-        root_certificates=root_certificates,
-        insecure=args.insecure,
-        authentication_keys=authentication_keys,
-        max_retries=args.max_retries,
-        max_wait_time=args.max_wait_time,
-    )
-    register_exit_handlers(event_type=EventType.RUN_CLIENT_APP_LEAVE)
-
 
 def _get_certificates(args: argparse.Namespace) -> Optional[bytes]:
     """Load certificates if specified in args."""
@@ -140,24 +158,88 @@ def _get_certificates(args: argparse.Namespace) -> Optional[bytes]:
 
 
 def _get_load_client_app_fn(
-    args: argparse.Namespace,
-) -> Callable[[], ClientApp]:
-    """Get the load_client_app_fn function."""
-    client_app_dir = args.dir
-    if client_app_dir is not None:
-        sys.path.insert(0, client_app_dir)
+    args: argparse.Namespace, multi_app: bool
+) -> Callable[[str, str], ClientApp]:
+    """Get the load_client_app_fn function.
+
+    If `multi_app` is True, this function loads the specified ClientApp
+    based on `fab_id` and `fab_version`. If `fab_id` is empty, a default
+    ClientApp will be loaded.
+
+    If `multi_app` is False, it ignores `fab_id` and `fab_version` and
+    loads a default ClientApp.
+    """
+    # Find the Flower directory containing Flower Apps (only for multi-app)
+    flwr_dir = Path("")
+    if "flwr_dir" in args:
+        if args.flwr_dir is None:
+            flwr_dir = get_flwr_dir()
+        else:
+            flwr_dir = Path(args.flwr_dir).absolute()
 
-    app_ref: str = getattr(args, "client-app")
-    valid, error_msg = validate(app_ref)
-    if not valid and error_msg:
-        raise LoadClientAppError(error_msg) from None
+    sys.path.insert(0, str(flwr_dir.absolute()))
 
-    def _load() -> ClientApp:
-        client_app = load_app(app_ref, LoadClientAppError)
+    default_app_ref: str = getattr(args, "client-app")
+
+    if not multi_app:
+        log(
+            DEBUG,
+            "Flower SuperNode will load and validate ClientApp `%s`",
+            getattr(args, "client-app"),
+        )
+        valid, error_msg = validate(default_app_ref)
+        if not valid and error_msg:
+            raise LoadClientAppError(error_msg) from None
+
+    def _load(fab_id: str, fab_version: str) -> ClientApp:
+        # If multi-app feature is disabled
+        if not multi_app:
+            # Get sys path to be inserted
+            sys_path = Path(args.dir).absolute()
+
+            # Set app reference
+            client_app_ref = default_app_ref
+        # If multi-app feature is enabled but the fab id is not specified
+        elif fab_id == "":
+            if default_app_ref == "":
+                raise LoadClientAppError(
+                    "Invalid FAB ID: The FAB ID is empty.",
+                ) from None
+
+            log(WARN, "FAB ID is not provided; the default ClientApp will be loaded.")
+            # Get sys path to be inserted
+            sys_path = Path(args.dir).absolute()
+
+            # Set app reference
+            client_app_ref = default_app_ref
+        # If multi-app feature is enabled
+        else:
+            try:
+                project_dir = get_project_dir(fab_id, fab_version, flwr_dir)
+                config = get_project_config(project_dir)
+            except Exception as e:
+                raise LoadClientAppError("Failed to load ClientApp") from e
+
+            # Get sys path to be inserted
+            sys_path = Path(project_dir).absolute()
+
+            # Set app reference
+            client_app_ref = config["flower"]["components"]["clientapp"]
+
+        # Set sys.path
+        sys.path.insert(0, str(sys_path))
+
+        # Load ClientApp
+        log(
+            DEBUG,
+            "Loading ClientApp `%s`",
+            client_app_ref,
+        )
+        client_app = load_app(client_app_ref, LoadClientAppError, sys_path)
 
         if not isinstance(client_app, ClientApp):
             raise LoadClientAppError(
-                f"Attribute {app_ref} is not of type {ClientApp}",
+                f"Attribute {client_app_ref} is not of type {ClientApp}",
             ) from None
 
         return client_app
@@ -218,9 +300,27 @@ def _parse_args_common(parser: argparse.ArgumentParser) -> None:
         help="Run the client without HTTPS. By default, the client runs with "
         "HTTPS enabled. Use this flag only if you understand the risks.",
     )
-    parser.add_argument(
+    ex_group = parser.add_mutually_exclusive_group()
+    ex_group.add_argument(
+        "--grpc-rere",
+        action="store_const",
+        dest="transport",
+        const=TRANSPORT_TYPE_GRPC_RERE,
+        default=TRANSPORT_TYPE_GRPC_RERE,
+        help="Use grpc-rere as a transport layer for the client.",
+    )
+    ex_group.add_argument(
+        "--grpc-adapter",
+        action="store_const",
+        dest="transport",
+        const=TRANSPORT_TYPE_GRPC_ADAPTER,
+        help="Use grpc-adapter as a transport layer for the client.",
+    )
+    ex_group.add_argument(
         "--rest",
-        action="store_true",
+        action="store_const",
+        dest="transport",
+        const=TRANSPORT_TYPE_REST,
         help="Use REST as a transport layer for the client.",
     )
     parser.add_argument(

flwr/common/__init__.py

@@ -63,43 +63,34 @@ from .typing import Status as Status
 
 __all__ = [
     "Array",
-    "array_from_numpy",
-    "bytes_to_ndarray",
     "ClientMessage",
     "Code",
     "Config",
     "ConfigsRecord",
-    "configure",
     "Context",
+    "DEFAULT_TTL",
     "DisconnectRes",
+    "Error",
     "EvaluateIns",
     "EvaluateRes",
-    "event",
     "EventType",
     "FitIns",
     "FitRes",
-    "Error",
+    "GRPC_MAX_MESSAGE_LENGTH",
     "GetParametersIns",
     "GetParametersRes",
     "GetPropertiesIns",
     "GetPropertiesRes",
-    "GRPC_MAX_MESSAGE_LENGTH",
-    "log",
     "Message",
     "MessageType",
     "MessageTypeLegacy",
-    "DEFAULT_TTL",
     "Metadata",
     "Metrics",
     "MetricsAggregationFn",
     "MetricsRecord",
-    "ndarray_to_bytes",
-    "now",
     "NDArray",
     "NDArrays",
-    "ndarrays_to_parameters",
     "Parameters",
-    "parameters_to_ndarrays",
     "ParametersRecord",
     "Properties",
     "ReconnectIns",
@@ -107,4 +98,13 @@ __all__ = [
     "Scalar",
     "ServerMessage",
     "Status",
+    "array_from_numpy",
+    "bytes_to_ndarray",
+    "configure",
+    "event",
+    "log",
+    "ndarray_to_bytes",
+    "ndarrays_to_parameters",
+    "now",
+    "parameters_to_ndarrays",
 ]

flwr/common/config.py

@@ -0,0 +1,71 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Provide functions for managing global Flower config."""
+
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
+
+import tomli
+
+from flwr.cli.config_utils import validate_fields
+from flwr.common.constant import APP_DIR, FAB_CONFIG_FILE, FLWR_HOME
+
+
+def get_flwr_dir() -> Path:
+    """Return the Flower home directory based on env variables."""
+    return Path(
+        os.getenv(
+            FLWR_HOME,
+            f"{os.getenv('XDG_DATA_HOME', os.getenv('HOME'))}/.flwr",
+        )
+    )
+
+
+def get_project_dir(
+    fab_id: str, fab_version: str, flwr_dir: Optional[Union[str, Path]] = None
+) -> Path:
+    """Return the project directory based on the given fab_id and fab_version."""
+    # Check the fab_id
+    if fab_id.count("/") != 1:
+        raise ValueError(
+            f"Invalid FAB ID: {fab_id}",
+        )
+    publisher, project_name = fab_id.split("/")
+    if flwr_dir is None:
+        flwr_dir = get_flwr_dir()
+    return Path(flwr_dir) / APP_DIR / publisher / project_name / fab_version
+
+
+def get_project_config(project_dir: Union[str, Path]) -> Dict[str, Any]:
+    """Return pyproject.toml in the given project directory."""
+    # Load pyproject.toml file
+    toml_path = Path(project_dir) / FAB_CONFIG_FILE
+    if not toml_path.is_file():
+        raise FileNotFoundError(
+            f"Cannot find {FAB_CONFIG_FILE} in {project_dir}",
+        )
+    with toml_path.open(encoding="utf-8") as toml_file:
+        config = tomli.loads(toml_file.read())
+
+    # Validate pyproject.toml fields
+    is_valid, errors, _ = validate_fields(config)
+    if not is_valid:
+        error_msg = "\n".join([f"  - {error}" for error in errors])
+        raise ValueError(
+            f"Invalid {FAB_CONFIG_FILE}:\n{error_msg}",
+        )
+
+    return config

flwr/common/constant.py

@@ -27,6 +27,7 @@ To use the REST API, install `flwr` with the `rest` extra:
 
 TRANSPORT_TYPE_GRPC_BIDI = "grpc-bidi"
 TRANSPORT_TYPE_GRPC_RERE = "grpc-rere"
+TRANSPORT_TYPE_GRPC_ADAPTER = "grpc-adapter"
 TRANSPORT_TYPE_REST = "rest"
 TRANSPORT_TYPE_VCE = "vce"
 TRANSPORT_TYPES = [
@@ -36,6 +37,8 @@ TRANSPORT_TYPES = [
     TRANSPORT_TYPE_VCE,
 ]
 
+SUPEREXEC_DEFAULT_ADDRESS = "0.0.0.0:9093"
+
 # Constants for ping
 PING_DEFAULT_INTERVAL = 30
 PING_CALL_TIMEOUT = 5
@@ -43,6 +46,18 @@ PING_BASE_MULTIPLIER = 0.8
 PING_RANDOM_RANGE = (-0.1, 0.1)
 PING_MAX_INTERVAL = 1e300
 
+GRPC_ADAPTER_METADATA_FLOWER_VERSION_KEY = "flower-version"
+GRPC_ADAPTER_METADATA_SHOULD_EXIT_KEY = "should-exit"
+
+# Constants for FAB
+APP_DIR = "apps"
+FAB_CONFIG_FILE = "pyproject.toml"
+FLWR_HOME = "FLWR_HOME"
+
+
+GRPC_ADAPTER_METADATA_FLOWER_VERSION_KEY = "flower-version"
+GRPC_ADAPTER_METADATA_SHOULD_EXIT_KEY = "should-exit"
+
 
 class MessageType:
     """Message type."""

flwr/common/object_ref.py

@@ -17,8 +17,13 @@
 
 import ast
 import importlib
+import sys
 from importlib.util import find_spec
-from typing import Any, Optional, Tuple, Type
+from logging import WARN
+from pathlib import Path
+from typing import Any, Optional, Tuple, Type, Union
+
+from .logger import log
 
 OBJECT_REF_HELP_STR = """
 \n\nThe object reference string should have the form <module>:<attribute>. Valid
@@ -77,9 +82,10 @@ def validate(
     )
 
 
-def load_app(
+def load_app(  # pylint: disable= too-many-branches
     module_attribute_str: str,
     error_type: Type[Exception],
+    project_dir: Optional[Union[str, Path]] = None,
 ) -> Any:
     """Return the object specified in a module attribute string.
 
@@ -95,11 +101,39 @@ def load_app(
     module_str, _, attributes_str = module_attribute_str.partition(":")
 
     try:
-        module = importlib.import_module(module_str)
-    except ModuleNotFoundError:
+        if module_str not in sys.modules:
+            module = importlib.import_module(module_str)
+        # Hack: `tabnet` does not work with `importlib.reload`
+        elif "tabnet" in sys.modules:
+            log(
+                WARN,
+                "Cannot reload module `%s` from disk due to compatibility issues "
+                "with the `tabnet` library. The module will be loaded from the "
+                "cache instead. If you experience issues, consider restarting "
+                "the application.",
+                module_str,
+            )
+            module = sys.modules[module_str]
+        else:
+            module = sys.modules[module_str]
+            if project_dir is None:
+                path: Optional[str] = getattr(module, "__file__", None)
+                if path is not None:
+                    project_dir = str(Path(path).parent)
+            else:
+                project_dir = str(Path(project_dir).absolute())
+
+            # Reload cached modules in the project directory
+            if project_dir is not None:
+                for m in list(sys.modules.values()):
+                    path = getattr(m, "__file__", None)
+                    if path is not None and path.startswith(project_dir):
+                        importlib.reload(m)
+
+    except ModuleNotFoundError as err:
         raise error_type(
             f"Unable to load module {module_str}{OBJECT_REF_HELP_STR}",
-        ) from None
+        ) from err
 
     # Recursively load attribute
     attribute = module

flwr/common/record/__init__.py

@@ -22,9 +22,9 @@ from .recordset import RecordSet
 
 __all__ = [
     "Array",
-    "array_from_numpy",
     "ConfigsRecord",
     "MetricsRecord",
     "ParametersRecord",
     "RecordSet",
+    "array_from_numpy",
 ]

flwr/common/telemetry.py

@@ -164,6 +164,10 @@ class EventType(str, Enum):
     RUN_SUPERNODE_ENTER = auto()
     RUN_SUPERNODE_LEAVE = auto()
 
+    # SuperExec
+    RUN_SUPEREXEC_ENTER = auto()
+    RUN_SUPEREXEC_LEAVE = auto()
+
 
 # Use the ThreadPoolExecutor with max_workers=1 to have a queue
 # and also ensure that telemetry calls are not blocking.

flwr/common/typing.py

@@ -185,3 +185,12 @@ class ClientMessage:
     get_parameters_res: Optional[GetParametersRes] = None
     fit_res: Optional[FitRes] = None
     evaluate_res: Optional[EvaluateRes] = None
+
+
+@dataclass
+class Run:
+    """Run details."""
+
+    run_id: int
+    fab_id: str
+    fab_version: str