flwr-nightly 1.8.0.dev20240315__py3-none-any.whl → 1.11.0.dev20240813__py3-none-any.whl

flwr/client/supernode/app.py

@@ -0,0 +1,464 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower SuperNode."""
+
+import argparse
+import sys
+from logging import DEBUG, INFO, WARN
+from pathlib import Path
+from typing import Callable, Optional, Tuple
+
+from cryptography.exceptions import UnsupportedAlgorithm
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives.serialization import (
+    load_ssh_private_key,
+    load_ssh_public_key,
+)
+
+from flwr.client.client_app import ClientApp, LoadClientAppError
+from flwr.common import EventType, event
+from flwr.common.config import (
+    get_flwr_dir,
+    get_metadata_from_config,
+    get_project_config,
+    get_project_dir,
+    parse_config_args,
+)
+from flwr.common.constant import (
+    TRANSPORT_TYPE_GRPC_ADAPTER,
+    TRANSPORT_TYPE_GRPC_RERE,
+    TRANSPORT_TYPE_REST,
+)
+from flwr.common.exit_handlers import register_exit_handlers
+from flwr.common.logger import log, warn_deprecated_feature
+from flwr.common.object_ref import load_app, validate
+
+from ..app import _start_client_internal
+
+ADDRESS_FLEET_API_GRPC_RERE = "0.0.0.0:9092"
+
+
+def run_supernode() -> None:
+    """Run Flower SuperNode."""
+    log(INFO, "Starting Flower SuperNode")
+
+    event(EventType.RUN_SUPERNODE_ENTER)
+
+    args = _parse_args_run_supernode().parse_args()
+
+    _warn_deprecated_server_arg(args)
+
+    root_certificates = _get_certificates(args)
+    load_fn = _get_load_client_app_fn(
+        default_app_ref="",
+        app_path=args.app,
+        flwr_dir=args.flwr_dir,
+        multi_app=True,
+    )
+    authentication_keys = _try_setup_client_authentication(args)
+
+    _start_client_internal(
+        server_address=args.superlink,
+        load_client_app_fn=load_fn,
+        transport=args.transport,
+        root_certificates=root_certificates,
+        insecure=args.insecure,
+        authentication_keys=authentication_keys,
+        max_retries=args.max_retries,
+        max_wait_time=args.max_wait_time,
+        node_config=parse_config_args([args.node_config]),
+        flwr_path=get_flwr_dir(args.flwr_dir),
+    )
+
+    # Graceful shutdown
+    register_exit_handlers(
+        event_type=EventType.RUN_SUPERNODE_LEAVE,
+    )
+
+
+def run_client_app() -> None:
+    """Run Flower client app."""
+    log(INFO, "Long-running Flower client starting")
+
+    event(EventType.RUN_CLIENT_APP_ENTER)
+
+    args = _parse_args_run_client_app().parse_args()
+
+    _warn_deprecated_server_arg(args)
+
+    root_certificates = _get_certificates(args)
+    load_fn = _get_load_client_app_fn(
+        default_app_ref=getattr(args, "client-app"),
+        app_path=args.dir,
+        multi_app=False,
+    )
+    authentication_keys = _try_setup_client_authentication(args)
+
+    _start_client_internal(
+        server_address=args.superlink,
+        node_config=parse_config_args([args.node_config]),
+        load_client_app_fn=load_fn,
+        transport=args.transport,
+        root_certificates=root_certificates,
+        insecure=args.insecure,
+        authentication_keys=authentication_keys,
+        max_retries=args.max_retries,
+        max_wait_time=args.max_wait_time,
+    )
+    register_exit_handlers(event_type=EventType.RUN_CLIENT_APP_LEAVE)
+
+
+def flwr_clientapp() -> None:
+    """Run process-isolated Flower ClientApp."""
+    log(INFO, "Starting Flower ClientApp")
+
+    parser = argparse.ArgumentParser(
+        description="Run a Flower ClientApp",
+    )
+    parser.add_argument(
+        "--address",
+        help="Address of SuperNode ClientAppIo gRPC servicer",
+    )
+    parser.add_argument(
+        "--token",
+        help="Unique token generated by SuperNode for each ClientApp execution",
+    )
+    args = parser.parse_args()
+    log(
+        DEBUG,
+        "Staring isolated `ClientApp` connected to SuperNode ClientAppIo at %s "
+        "with the token %s",
+        args.address,
+        args.token,
+    )
+
+
+def _warn_deprecated_server_arg(args: argparse.Namespace) -> None:
+    """Warn about the deprecated argument `--server`."""
+    if args.server != ADDRESS_FLEET_API_GRPC_RERE:
+        warn = "Passing flag --server is deprecated. Use --superlink instead."
+        warn_deprecated_feature(warn)
+
+        if args.superlink != ADDRESS_FLEET_API_GRPC_RERE:
+            # if `--superlink` also passed, then
+            # warn user that this argument overrides what was passed with `--server`
+            log(
+                WARN,
+                "Both `--server` and `--superlink` were passed. "
+                "`--server` will be ignored. Connecting to the Superlink Fleet API "
+                "at %s.",
+                args.superlink,
+            )
+        else:
+            args.superlink = args.server
+
+
+def _get_certificates(args: argparse.Namespace) -> Optional[bytes]:
+    """Load certificates if specified in args."""
+    # Obtain certificates
+    if args.insecure:
+        if args.root_certificates is not None:
+            sys.exit(
+                "Conflicting options: The '--insecure' flag disables HTTPS, "
+                "but '--root-certificates' was also specified. Please remove "
+                "the '--root-certificates' option when running in insecure mode, "
+                "or omit '--insecure' to use HTTPS."
+            )
+        log(
+            WARN,
+            "Option `--insecure` was set. "
+            "Starting insecure HTTP client connected to %s.",
+            args.superlink,
+        )
+        root_certificates = None
+    else:
+        # Load the certificates if provided, or load the system certificates
+        cert_path = args.root_certificates
+        if cert_path is None:
+            root_certificates = None
+        else:
+            root_certificates = Path(cert_path).read_bytes()
+        log(
+            DEBUG,
+            "Starting secure HTTPS client connected to %s "
+            "with the following certificates: %s.",
+            args.superlink,
+            cert_path,
+        )
+    return root_certificates
+
+
+def _get_load_client_app_fn(
+    default_app_ref: str,
+    app_path: Optional[str],
+    multi_app: bool,
+    flwr_dir: Optional[str] = None,
+) -> Callable[[str, str], ClientApp]:
+    """Get the load_client_app_fn function.
+
+    If `multi_app` is True, this function loads the specified ClientApp
+    based on `fab_id` and `fab_version`. If `fab_id` is empty, a default
+    ClientApp will be loaded.
+
+    If `multi_app` is False, it ignores `fab_id` and `fab_version` and
+    loads a default ClientApp.
+    """
+    if not multi_app:
+        log(
+            DEBUG,
+            "Flower SuperNode will load and validate ClientApp `%s`",
+            default_app_ref,
+        )
+
+        valid, error_msg = validate(default_app_ref, project_dir=app_path)
+        if not valid and error_msg:
+            raise LoadClientAppError(error_msg) from None
+
+    def _load(fab_id: str, fab_version: str) -> ClientApp:
+        runtime_app_dir = Path(app_path if app_path else "").absolute()
+        # If multi-app feature is disabled
+        if not multi_app:
+            # Set app reference
+            client_app_ref = default_app_ref
+        # If multi-app feature is enabled but app directory is provided
+        elif app_path is not None:
+            config = get_project_config(runtime_app_dir)
+            this_fab_version, this_fab_id = get_metadata_from_config(config)
+
+            if this_fab_version != fab_version or this_fab_id != fab_id:
+                raise LoadClientAppError(
+                    f"FAB ID or version mismatch: Expected FAB ID '{this_fab_id}' and "
+                    f"FAB version '{this_fab_version}', but received FAB ID '{fab_id}' "
+                    f"and FAB version '{fab_version}'.",
+                ) from None
+
+            # log(WARN, "FAB ID is not provided; the default ClientApp will be loaded.")
+
+            # Set app reference
+            client_app_ref = config["tool"]["flwr"]["app"]["components"]["clientapp"]
+        # If multi-app feature is enabled
+        else:
+            try:
+                runtime_app_dir = get_project_dir(
+                    fab_id, fab_version, get_flwr_dir(flwr_dir)
+                )
+                config = get_project_config(runtime_app_dir)
+            except Exception as e:
+                raise LoadClientAppError("Failed to load ClientApp") from e
+
+            # Set app reference
+            client_app_ref = config["tool"]["flwr"]["app"]["components"]["clientapp"]
+
+        # Load ClientApp
+        log(
+            DEBUG,
+            "Loading ClientApp `%s`",
+            client_app_ref,
+        )
+        client_app = load_app(client_app_ref, LoadClientAppError, runtime_app_dir)
+
+        if not isinstance(client_app, ClientApp):
+            raise LoadClientAppError(
+                f"Attribute {client_app_ref} is not of type {ClientApp}",
+            ) from None
+
+        return client_app
+
+    return _load
+
+
+def _parse_args_run_supernode() -> argparse.ArgumentParser:
+    """Parse flower-supernode command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Start a Flower SuperNode",
+    )
+
+    parser.add_argument(
+        "app",
+        nargs="?",
+        default=None,
+        help="Specify the path of the Flower App to load and run the `ClientApp`. "
+        "The `pyproject.toml` file must be located in the root of this path. "
+        "When this argument is provided, the SuperNode will exclusively respond to "
+        "messages from the corresponding `ServerApp` by matching the FAB ID and FAB "
+        "version. An error will be raised if a message is received from any other "
+        "`ServerApp`.",
+    )
+    _parse_args_common(parser)
+    parser.add_argument(
+        "--flwr-dir",
+        default=None,
+        help="""The path containing installed Flower Apps.
+    By default, this value is equal to:
+
+        - `$FLWR_HOME/` if `$FLWR_HOME` is defined
+        - `$XDG_DATA_HOME/.flwr/` if `$XDG_DATA_HOME` is defined
+        - `$HOME/.flwr/` in all other cases
+    """,
+    )
+
+    return parser
+
+
+def _parse_args_run_client_app() -> argparse.ArgumentParser:
+    """Parse flower-client-app command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Start a Flower client app",
+    )
+
+    parser.add_argument(
+        "client-app",
+        help="For example: `client:app` or `project.package.module:wrapper.app`",
+    )
+    _parse_args_common(parser=parser)
+    parser.add_argument(
+        "--dir",
+        default="",
+        help="Add specified directory to the PYTHONPATH and load Flower "
+        "app from there."
+        " Default: current working directory.",
+    )
+
+    return parser
+
+
+def _parse_args_common(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--insecure",
+        action="store_true",
+        help="Run the client without HTTPS. By default, the client runs with "
+        "HTTPS enabled. Use this flag only if you understand the risks.",
+    )
+    ex_group = parser.add_mutually_exclusive_group()
+    ex_group.add_argument(
+        "--grpc-rere",
+        action="store_const",
+        dest="transport",
+        const=TRANSPORT_TYPE_GRPC_RERE,
+        default=TRANSPORT_TYPE_GRPC_RERE,
+        help="Use grpc-rere as a transport layer for the client.",
+    )
+    ex_group.add_argument(
+        "--grpc-adapter",
+        action="store_const",
+        dest="transport",
+        const=TRANSPORT_TYPE_GRPC_ADAPTER,
+        help="Use grpc-adapter as a transport layer for the client.",
+    )
+    ex_group.add_argument(
+        "--rest",
+        action="store_const",
+        dest="transport",
+        const=TRANSPORT_TYPE_REST,
+        help="Use REST as a transport layer for the client.",
+    )
+    parser.add_argument(
+        "--root-certificates",
+        metavar="ROOT_CERT",
+        type=str,
+        help="Specifies the path to the PEM-encoded root certificate file for "
+        "establishing secure HTTPS connections.",
+    )
+    parser.add_argument(
+        "--server",
+        default=ADDRESS_FLEET_API_GRPC_RERE,
+        help="Server address",
+    )
+    parser.add_argument(
+        "--superlink",
+        default=ADDRESS_FLEET_API_GRPC_RERE,
+        help="SuperLink Fleet API (gRPC-rere) address (IPv4, IPv6, or a domain name)",
+    )
+    parser.add_argument(
+        "--max-retries",
+        type=int,
+        default=None,
+        help="The maximum number of times the client will try to reconnect to the"
+        "SuperLink before giving up in case of a connection error. By default,"
+        "it is set to None, meaning there is no limit to the number of tries.",
+    )
+    parser.add_argument(
+        "--max-wait-time",
+        type=float,
+        default=None,
+        help="The maximum duration before the client stops trying to"
+        "connect to the SuperLink in case of connection error. By default, it"
+        "is set to None, meaning there is no limit to the total time.",
+    )
+    parser.add_argument(
+        "--auth-supernode-private-key",
+        type=str,
+        help="The SuperNode's private key (as a path str) to enable authentication.",
+    )
+    parser.add_argument(
+        "--auth-supernode-public-key",
+        type=str,
+        help="The SuperNode's public key (as a path str) to enable authentication.",
+    )
+    parser.add_argument(
+        "--node-config",
+        type=str,
+        help="A comma separated list of key/value pairs (separated by `=`) to "
+        "configure the SuperNode. "
+        "E.g. --node-config 'key1=\"value1\",partition-id=0,num-partitions=100'",
+    )
+
+
+def _try_setup_client_authentication(
+    args: argparse.Namespace,
+) -> Optional[Tuple[ec.EllipticCurvePrivateKey, ec.EllipticCurvePublicKey]]:
+    if not args.auth_supernode_private_key and not args.auth_supernode_public_key:
+        return None
+
+    if not args.auth_supernode_private_key or not args.auth_supernode_public_key:
+        sys.exit(
+            "Authentication requires file paths to both "
+            "'--auth-supernode-private-key' and '--auth-supernode-public-key'"
+            "to be provided (providing only one of them is not sufficient)."
+        )
+
+    try:
+        ssh_private_key = load_ssh_private_key(
+            Path(args.auth_supernode_private_key).read_bytes(),
+            None,
+        )
+        if not isinstance(ssh_private_key, ec.EllipticCurvePrivateKey):
+            raise ValueError()
+    except (ValueError, UnsupportedAlgorithm):
+        sys.exit(
+            "Error: Unable to parse the private key file in "
+            "'--auth-supernode-private-key'. Authentication requires elliptic "
+            "curve private and public key pair. Please ensure that the file "
+            "path points to a valid private key file and try again."
+        )
+
+    try:
+        ssh_public_key = load_ssh_public_key(
+            Path(args.auth_supernode_public_key).read_bytes()
+        )
+        if not isinstance(ssh_public_key, ec.EllipticCurvePublicKey):
+            raise ValueError()
+    except (ValueError, UnsupportedAlgorithm):
+        sys.exit(
+            "Error: Unable to parse the public key file in "
+            "'--auth-supernode-public-key'. Authentication requires elliptic "
+            "curve private and public key pair. Please ensure that the file "
+            "path points to a valid public key file and try again."
+        )
+
+    return (
+        ssh_private_key,
+        ssh_public_key,
+    )

flwr/client/typing.py

@@ -23,6 +23,7 @@ from .client import Client as Client
 
 # Compatibility
 ClientFn = Callable[[str], Client]
+ClientFnExt = Callable[[Context], Client]
 
 ClientAppCallable = Callable[[Message, Context], Message]
 Mod = Callable[[Message, Context, ClientAppCallable], Message]

flwr/common/__init__.py

@@ -22,6 +22,7 @@ from .date import now as now
 from .grpc import GRPC_MAX_MESSAGE_LENGTH
 from .logger import configure as configure
 from .logger import log as log
+from .message import DEFAULT_TTL
 from .message import Error as Error
 from .message import Message as Message
 from .message import Metadata as Metadata
@@ -62,28 +63,24 @@ from .typing import Status as Status
 
 __all__ = [
     "Array",
-    "array_from_numpy",
-    "bytes_to_ndarray",
     "ClientMessage",
     "Code",
     "Config",
     "ConfigsRecord",
-    "configure",
     "Context",
+    "DEFAULT_TTL",
     "DisconnectRes",
+    "Error",
     "EvaluateIns",
     "EvaluateRes",
-    "event",
     "EventType",
     "FitIns",
     "FitRes",
-    "Error",
+    "GRPC_MAX_MESSAGE_LENGTH",
     "GetParametersIns",
     "GetParametersRes",
     "GetPropertiesIns",
     "GetPropertiesRes",
-    "GRPC_MAX_MESSAGE_LENGTH",
-    "log",
     "Message",
     "MessageType",
     "MessageTypeLegacy",
@@ -91,13 +88,9 @@ __all__ = [
     "Metrics",
     "MetricsAggregationFn",
     "MetricsRecord",
-    "ndarray_to_bytes",
-    "now",
     "NDArray",
     "NDArrays",
-    "ndarrays_to_parameters",
     "Parameters",
-    "parameters_to_ndarrays",
     "ParametersRecord",
     "Properties",
     "ReconnectIns",
@@ -105,4 +98,13 @@ __all__ = [
     "Scalar",
     "ServerMessage",
     "Status",
+    "array_from_numpy",
+    "bytes_to_ndarray",
+    "configure",
+    "event",
+    "log",
+    "ndarray_to_bytes",
+    "ndarrays_to_parameters",
+    "now",
+    "parameters_to_ndarrays",
 ]

flwr/common/address.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Flower Labs GmbH. All Rights Reserved.
+# Copyright 2023 Flower Labs GmbH. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.