flwr-nightly 1.9.0.dev20240509__py3-none-any.whl → 1.9.0.dev20240531__py3-none-any.whl

flwr/proto/grpcadapter_pb2_grpc.py

@@ -0,0 +1,66 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+from flwr.proto import grpcadapter_pb2 as flwr_dot_proto_dot_grpcadapter__pb2
+
+
+class GrpcAdapterStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.SendReceive = channel.unary_unary(
+                '/flwr.proto.GrpcAdapter/SendReceive',
+                request_serializer=flwr_dot_proto_dot_grpcadapter__pb2.MessageContainer.SerializeToString,
+                response_deserializer=flwr_dot_proto_dot_grpcadapter__pb2.MessageContainer.FromString,
+                )
+
+
+class GrpcAdapterServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def SendReceive(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_GrpcAdapterServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'SendReceive': grpc.unary_unary_rpc_method_handler(
+                    servicer.SendReceive,
+                    request_deserializer=flwr_dot_proto_dot_grpcadapter__pb2.MessageContainer.FromString,
+                    response_serializer=flwr_dot_proto_dot_grpcadapter__pb2.MessageContainer.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'flwr.proto.GrpcAdapter', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class GrpcAdapter(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def SendReceive(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/flwr.proto.GrpcAdapter/SendReceive',
+            flwr_dot_proto_dot_grpcadapter__pb2.MessageContainer.SerializeToString,
+            flwr_dot_proto_dot_grpcadapter__pb2.MessageContainer.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

flwr/proto/grpcadapter_pb2_grpc.pyi

@@ -0,0 +1,24 @@
+"""
+@generated by mypy-protobuf.  Do not edit manually!
+isort:skip_file
+"""
+import abc
+import flwr.proto.grpcadapter_pb2
+import grpc
+
+class GrpcAdapterStub:
+    def __init__(self, channel: grpc.Channel) -> None: ...
+    SendReceive: grpc.UnaryUnaryMultiCallable[
+        flwr.proto.grpcadapter_pb2.MessageContainer,
+        flwr.proto.grpcadapter_pb2.MessageContainer]
+
+
+class GrpcAdapterServicer(metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def SendReceive(self,
+        request: flwr.proto.grpcadapter_pb2.MessageContainer,
+        context: grpc.ServicerContext,
+    ) -> flwr.proto.grpcadapter_pb2.MessageContainer: ...
+
+
+def add_GrpcAdapterServicer_to_server(servicer: GrpcAdapterServicer, server: grpc.Server) -> None: ...

flwr/server/app.py

@@ -15,17 +15,17 @@
 """Flower server app."""
 
 import argparse
-import asyncio
 import csv
 import importlib.util
 import sys
 import threading
-from logging import ERROR, INFO, WARN
+from logging import INFO, WARN
 from os.path import isfile
 from pathlib import Path
-from typing import List, Optional, Sequence, Set, Tuple
+from typing import Optional, Sequence, Set, Tuple
 
 import grpc
+from cryptography.exceptions import UnsupportedAlgorithm
 from cryptography.hazmat.primitives.asymmetric import ec
 from cryptography.hazmat.primitives.serialization import (
     load_ssh_private_key,
@@ -38,14 +38,12 @@ from flwr.common.constant import (
     MISSING_EXTRA_REST,
     TRANSPORT_TYPE_GRPC_RERE,
     TRANSPORT_TYPE_REST,
-    TRANSPORT_TYPE_VCE,
 )
 from flwr.common.exit_handlers import register_exit_handlers
 from flwr.common.logger import log, warn_deprecated_feature
 from flwr.common.secure_aggregation.crypto.symmetric_encryption import (
     private_key_to_bytes,
     public_key_to_bytes,
-    ssh_types_to_elliptic_curve,
 )
 from flwr.proto.fleet_pb2_grpc import (  # pylint: disable=E0611
     add_FleetServicer_to_server,
@@ -63,7 +61,6 @@ from .superlink.fleet.grpc_bidi.grpc_server import (
 )
 from .superlink.fleet.grpc_rere.fleet_servicer import FleetServicer
 from .superlink.fleet.grpc_rere.server_interceptor import AuthenticateServerInterceptor
-from .superlink.fleet.vce import start_vce
 from .superlink.state import StateFactory
 
 ADDRESS_DRIVER_API = "0.0.0.0:9091"
@@ -349,6 +346,9 @@ def run_superlink() -> None:
             sys.exit(MISSING_EXTRA_REST)
         address_arg = args.rest_fleet_api_address
         parsed_address = parse_address(address_arg)
+        _, ssl_certfile, ssl_keyfile = (
+            certificates if certificates is not None else (None, None, None)
+        )
         if not parsed_address:
             sys.exit(f"Fleet IP address ({address_arg}) cannot be parsed.")
         host, port, _ = parsed_address
@@ -357,8 +357,8 @@ def run_superlink() -> None:
             args=(
                 host,
                 port,
-                args.ssl_keyfile,
-                args.ssl_certfile,
+                ssl_keyfile,
+                ssl_certfile,
                 state_factory,
                 args.rest_fleet_api_workers,
             ),
@@ -401,17 +401,6 @@ def run_superlink() -> None:
             interceptors=interceptors,
         )
         grpc_servers.append(fleet_server)
-    elif args.fleet_api_type == TRANSPORT_TYPE_VCE:
-        f_stop = asyncio.Event()  # Does nothing
-        _run_fleet_api_vce(
-            num_supernodes=args.num_supernodes,
-            client_app_attr=args.client_app,
-            backend_name=args.backend,
-            backend_config_json_stream=args.backend_config,
-            app_dir=args.app_dir,
-            state_factory=state_factory,
-            f_stop=f_stop,
-        )
     else:
         raise ValueError(f"Unknown fleet_api_type: {args.fleet_api_type}")
 
@@ -435,44 +424,69 @@ def _try_setup_client_authentication(
     args: argparse.Namespace,
     certificates: Optional[Tuple[bytes, bytes, bytes]],
 ) -> Optional[Tuple[Set[bytes], ec.EllipticCurvePrivateKey, ec.EllipticCurvePublicKey]]:
-    if not args.require_client_authentication:
+    if (
+        not args.auth_list_public_keys
+        and not args.auth_superlink_private_key
+        and not args.auth_superlink_public_key
+    ):
         return None
 
+    if (
+        not args.auth_list_public_keys
+        or not args.auth_superlink_private_key
+        or not args.auth_superlink_public_key
+    ):
+        sys.exit(
+            "Authentication requires providing file paths for "
+            "'--auth-list-public-keys', '--auth-superlink-private-key' and "
+            "'--auth-superlink-public-key'. Provide all three to enable authentication."
+        )
+
     if certificates is None:
         sys.exit(
-            "Client authentication only works over secure connections. "
-            "Please provide certificate paths using '--certificates' when "
-            "enabling '--require-client-authentication'."
+            "Authentication requires secure connections. "
+            "Please provide certificate paths to `--ssl-certfile`, "
+            "`--ssl-keyfile`, and `—-ssl-ca-certfile` and try again."
         )
 
-    client_keys_file_path = Path(args.require_client_authentication[0])
+    client_keys_file_path = Path(args.auth_list_public_keys)
     if not client_keys_file_path.exists():
         sys.exit(
-            "The provided path to the client public keys CSV file does not exist: "
+            "The provided path to the known public keys CSV file does not exist: "
             f"{client_keys_file_path}. "
-            "Please provide the CSV file path containing known client public keys "
-            "to '--require-client-authentication'."
+            "Please provide the CSV file path containing known public keys "
+            "to '--auth-list-public-keys'."
         )
 
     client_public_keys: Set[bytes] = set()
-    ssh_private_key = load_ssh_private_key(
-        Path(args.require_client_authentication[1]).read_bytes(),
-        None,
-    )
-    ssh_public_key = load_ssh_public_key(
-        Path(args.require_client_authentication[2]).read_bytes()
-    )
 
     try:
-        server_private_key, server_public_key = ssh_types_to_elliptic_curve(
-            ssh_private_key, ssh_public_key
+        ssh_private_key = load_ssh_private_key(
+            Path(args.auth_superlink_private_key).read_bytes(),
+            None,
         )
-    except TypeError:
+        if not isinstance(ssh_private_key, ec.EllipticCurvePrivateKey):
+            raise ValueError()
+    except (ValueError, UnsupportedAlgorithm):
         sys.exit(
-            "The file paths provided could not be read as a private and public "
-            "key pair. Client authentication requires an elliptic curve public and "
-            "private key pair. Please provide the file paths containing elliptic "
-            "curve private and public keys to '--require-client-authentication'."
+            "Error: Unable to parse the private key file in "
+            "'--auth-superlink-private-key'. Authentication requires elliptic "
+            "curve private and public key pair. Please ensure that the file "
+            "path points to a valid private key file and try again."
+        )
+
+    try:
+        ssh_public_key = load_ssh_public_key(
+            Path(args.auth_superlink_public_key).read_bytes()
+        )
+        if not isinstance(ssh_public_key, ec.EllipticCurvePublicKey):
+            raise ValueError()
+    except (ValueError, UnsupportedAlgorithm):
+        sys.exit(
+            "Error: Unable to parse the public key file in "
+            "'--auth-superlink-public-key'. Authentication requires elliptic "
+            "curve private and public key pair. Please ensure that the file "
+            "path points to a valid public key file and try again."
         )
 
     with open(client_keys_file_path, newline="", encoding="utf-8") as csvfile:
@@ -484,14 +498,14 @@ def _try_setup_client_authentication(
                     client_public_keys.add(public_key_to_bytes(public_key))
                 else:
                     sys.exit(
-                        "Error: Unable to parse the public keys in the .csv "
-                        "file. Please ensure that the .csv file contains valid "
-                        "SSH public keys and try again."
+                        "Error: Unable to parse the public keys in the CSV "
+                        "file. Please ensure that the CSV file path points to a valid "
+                        "known SSH public keys files and try again."
                     )
         return (
             client_public_keys,
-            server_private_key,
-            server_public_key,
+            ssh_private_key,
+            ssh_public_key,
         )
 
 
@@ -501,21 +515,52 @@ def _try_obtain_certificates(
     # Obtain certificates
     if args.insecure:
         log(WARN, "Option `--insecure` was set. Starting insecure HTTP server.")
-        certificates = None
+        return None
     # Check if certificates are provided
-    elif args.certificates:
-        certificates = (
-            Path(args.certificates[0]).read_bytes(),  # CA certificate
-            Path(args.certificates[1]).read_bytes(),  # server certificate
-            Path(args.certificates[2]).read_bytes(),  # server private key
-        )
-    else:
-        sys.exit(
-            "Certificates are required unless running in insecure mode. "
-            "Please provide certificate paths with '--certificates' or run the server "
-            "in insecure mode using '--insecure' if you understand the risks."
-        )
-    return certificates
+    if args.fleet_api_type == TRANSPORT_TYPE_GRPC_RERE:
+        if args.ssl_certfile and args.ssl_keyfile and args.ssl_ca_certfile:
+            if not isfile(args.ssl_ca_certfile):
+                sys.exit("Path argument `--ssl-ca-certfile` does not point to a file.")
+            if not isfile(args.ssl_certfile):
+                sys.exit("Path argument `--ssl-certfile` does not point to a file.")
+            if not isfile(args.ssl_keyfile):
+                sys.exit("Path argument `--ssl-keyfile` does not point to a file.")
+            certificates = (
+                Path(args.ssl_ca_certfile).read_bytes(),  # CA certificate
+                Path(args.ssl_certfile).read_bytes(),  # server certificate
+                Path(args.ssl_keyfile).read_bytes(),  # server private key
+            )
+            return certificates
+        if args.ssl_certfile or args.ssl_keyfile or args.ssl_ca_certfile:
+            sys.exit(
+                "You need to provide valid file paths to `--ssl-certfile`, "
+                "`--ssl-keyfile`, and `—-ssl-ca-certfile` to create a secure "
+                "connection in Fleet API server (gRPC-rere)."
+            )
+    if args.fleet_api_type == TRANSPORT_TYPE_REST:
+        if args.ssl_certfile and args.ssl_keyfile:
+            if not isfile(args.ssl_certfile):
+                sys.exit("Path argument `--ssl-certfile` does not point to a file.")
+            if not isfile(args.ssl_keyfile):
+                sys.exit("Path argument `--ssl-keyfile` does not point to a file.")
+            certificates = (
+                b"",
+                Path(args.ssl_certfile).read_bytes(),  # server certificate
+                Path(args.ssl_keyfile).read_bytes(),  # server private key
+            )
+            return certificates
+        if args.ssl_certfile or args.ssl_keyfile:
+            sys.exit(
+                "You need to provide valid file paths to `--ssl-certfile` "
+                "and `--ssl-keyfile` to create a secure connection "
+                "in Fleet API server (REST, experimental)."
+            )
+    sys.exit(
+        "Certificates are required unless running in insecure mode. "
+        "Please provide certificate paths to `--ssl-certfile`, "
+        "`--ssl-keyfile`, and `—-ssl-ca-certfile` or run the server "
+        "in insecure mode using '--insecure' if you understand the risks."
+    )
 
 
 def _run_fleet_api_grpc_rere(
@@ -544,29 +589,6 @@ def _run_fleet_api_grpc_rere(
     return fleet_grpc_server
 
 
-# pylint: disable=too-many-arguments
-def _run_fleet_api_vce(
-    num_supernodes: int,
-    client_app_attr: str,
-    backend_name: str,
-    backend_config_json_stream: str,
-    app_dir: str,
-    state_factory: StateFactory,
-    f_stop: asyncio.Event,
-) -> None:
-    log(INFO, "Flower VCE: Starting Fleet API (VirtualClientEngine)")
-
-    start_vce(
-        num_supernodes=num_supernodes,
-        client_app_attr=client_app_attr,
-        backend_name=backend_name,
-        backend_config_json_stream=backend_config_json_stream,
-        state_factory=state_factory,
-        app_dir=app_dir,
-        f_stop=f_stop,
-    )
-
-
 # pylint: disable=import-outside-toplevel,too-many-arguments
 def _run_fleet_api_rest(
     host: str,
@@ -594,14 +616,6 @@ def _run_fleet_api_rest(
     # See: https://www.starlette.io/applications/#accessing-the-app-instance
     fast_api_app.state.STATE_FACTORY = state_factory
 
-    validation_exceptions = _validate_ssl_files(
-        ssl_certfile=ssl_certfile, ssl_keyfile=ssl_keyfile
-    )
-    if any(validation_exceptions):
-        # Starting with 3.11 we can use ExceptionGroup but for now
-        # this seems to be the reasonable approach.
-        raise ValueError(validation_exceptions)
-
     uvicorn.run(
         app="flwr.server.superlink.fleet.rest_rere.rest_api:app",
         port=port,
@@ -614,32 +628,6 @@ def _run_fleet_api_rest(
     )
 
 
-def _validate_ssl_files(
-    ssl_keyfile: Optional[str], ssl_certfile: Optional[str]
-) -> List[ValueError]:
-    validation_exceptions = []
-
-    if ssl_keyfile is not None and not isfile(ssl_keyfile):
-        msg = "Path argument `--ssl-keyfile` does not point to a file."
-        log(ERROR, msg)
-        validation_exceptions.append(ValueError(msg))
-
-    if ssl_certfile is not None and not isfile(ssl_certfile):
-        msg = "Path argument `--ssl-certfile` does not point to a file."
-        log(ERROR, msg)
-        validation_exceptions.append(ValueError(msg))
-
-    if not bool(ssl_keyfile) == bool(ssl_certfile):
-        msg = (
-            "When setting one of `--ssl-keyfile` and "
-            "`--ssl-certfile`, both have to be used."
-        )
-        log(ERROR, msg)
-        validation_exceptions.append(ValueError(msg))
-
-    return validation_exceptions
-
-
 def _parse_args_run_driver_api() -> argparse.ArgumentParser:
     """Parse command line arguments for Driver API."""
     parser = argparse.ArgumentParser(
@@ -696,13 +684,23 @@ def _add_args_common(parser: argparse.ArgumentParser) -> None:
         "Use this flag only if you understand the risks.",
     )
     parser.add_argument(
-        "--certificates",
-        nargs=3,
-        metavar=("CA_CERT", "SERVER_CERT", "PRIVATE_KEY"),
+        "--ssl-certfile",
+        help="Fleet API server SSL certificate file (as a path str) "
+        "to create a secure connection.",
+        type=str,
+        default=None,
+    )
+    parser.add_argument(
+        "--ssl-keyfile",
+        help="Fleet API server SSL private key file (as a path str) "
+        "to create a secure connection.",
+        type=str,
+    )
+    parser.add_argument(
+        "--ssl-ca-certfile",
+        help="Fleet API server SSL CA certificate file (as a path str) "
+        "to create a secure connection.",
         type=str,
-        help="Paths to the CA certificate, server certificate, and server private "
-        "key, in that order. Note: The server can only be started without "
-        "certificates by enabling the `--insecure` flag.",
     )
     parser.add_argument(
         "--database",
@@ -714,13 +712,20 @@ def _add_args_common(parser: argparse.ArgumentParser) -> None:
         default=DATABASE,
     )
     parser.add_argument(
-        "--require-client-authentication",
-        nargs=3,
-        metavar=("CLIENT_KEYS", "SERVER_PRIVATE_KEY", "SERVER_PUBLIC_KEY"),
+        "--auth-list-public-keys",
+        type=str,
+        help="A CSV file (as a path str) containing a list of known public "
+        "keys to enable authentication.",
+    )
+    parser.add_argument(
+        "--auth-superlink-private-key",
+        type=str,
+        help="The SuperLink's private key (as a path str) to enable authentication.",
+    )
+    parser.add_argument(
+        "--auth-superlink-public-key",
         type=str,
-        help="Provide three file paths: (1) a .csv file containing a list of "
-        "known client public keys for authentication, (2) the server's private "
-        "key file, and (3) the server's public key file.",
+        help="The SuperLink's public key (as a path str) to enable authentication.",
     )
 
 
@@ -751,14 +756,6 @@ def _add_args_fleet_api(parser: argparse.ArgumentParser) -> None:
         help="Start a Fleet API server (REST, experimental)",
     )
 
-    ex_group.add_argument(
-        "--vce",
-        action="store_const",
-        dest="fleet_api_type",
-        const=TRANSPORT_TYPE_VCE,
-        help="Start a Fleet API server (VirtualClientEngine)",
-    )
-
     # Fleet API gRPC-rere options
     grpc_rere_group = parser.add_argument_group(
         "Fleet API (gRPC-rere) server options", ""
@@ -776,54 +773,9 @@ def _add_args_fleet_api(parser: argparse.ArgumentParser) -> None:
         help="Fleet API (REST) server address (IPv4, IPv6, or a domain name)",
         default=ADDRESS_FLEET_API_REST,
     )
-    rest_group.add_argument(
-        "--ssl-certfile",
-        help="Fleet API (REST) server SSL certificate file (as a path str), "
-        "needed for using 'https'.",
-        default=None,
-    )
-    rest_group.add_argument(
-        "--ssl-keyfile",
-        help="Fleet API (REST) server SSL private key file (as a path str), "
-        "needed for using 'https'.",
-        default=None,
-    )
     rest_group.add_argument(
         "--rest-fleet-api-workers",
         help="Set the number of concurrent workers for the Fleet API REST server.",
         type=int,
         default=1,
     )
-
-    # Fleet API VCE options
-    vce_group = parser.add_argument_group("Fleet API (VCE) server options", "")
-    vce_group.add_argument(
-        "--client-app",
-        help="For example: `client:app` or `project.package.module:wrapper.app`.",
-    )
-    vce_group.add_argument(
-        "--num-supernodes",
-        type=int,
-        help="Number of simulated SuperNodes.",
-    )
-    vce_group.add_argument(
-        "--backend",
-        default="ray",
-        type=str,
-        help="Simulation backend that executes the ClientApp.",
-    )
-    vce_group.add_argument(
-        "--backend-config",
-        type=str,
-        default='{"client_resources": {"num_cpus":1, "num_gpus":0.0}, "tensorflow": 0}',
-        help='A JSON formatted stream, e.g \'{"<keyA>":<value>, "<keyB>":<value>}\' to '
-        "configure a backend. Values supported in <value> are those included by "
-        "`flwr.common.typing.ConfigsRecordValues`. ",
-    )
-    parser.add_argument(
-        "--app-dir",
-        default="",
-        help="Add specified directory to the PYTHONPATH and load"
-        "ClientApp from there."
-        " Default: current working directory.",
-    )

flwr/server/driver/__init__.py

@@ -16,10 +16,11 @@
 
 
 from .driver import Driver
-from .grpc_driver import GrpcDriver, GrpcDriverHelper
+from .grpc_driver import GrpcDriver
+from .inmemory_driver import InMemoryDriver
 
 __all__ = [
     "Driver",
     "GrpcDriver",
-    "GrpcDriverHelper",
+    "InMemoryDriver",
 ]