flwr-nightly 1.8.0.dev20240315__py3-none-any.whl → 1.15.0.dev20250114__py3-none-any.whl

flwr/client/run_info_store.py

@@ -0,0 +1,112 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Deprecated Run Info Store."""
+
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+from flwr.common import Context, RecordSet
+from flwr.common.config import (
+    get_fused_config,
+    get_fused_config_from_dir,
+    get_fused_config_from_fab,
+)
+from flwr.common.typing import Fab, Run, UserConfig
+
+
+@dataclass()
+class RunInfo:
+    """Contains the Context and initial run_config of a Run."""
+
+    context: Context
+    initial_run_config: UserConfig
+
+
+class DeprecatedRunInfoStore:
+    """State of a node where client nodes execute runs."""
+
+    def __init__(
+        self,
+        node_id: int,
+        node_config: UserConfig,
+    ) -> None:
+        self.node_id = node_id
+        self.node_config = node_config
+        self.run_infos: dict[int, RunInfo] = {}
+
+    # pylint: disable=too-many-arguments,too-many-positional-arguments
+    def register_context(
+        self,
+        run_id: int,
+        run: Optional[Run] = None,
+        flwr_path: Optional[Path] = None,
+        app_dir: Optional[str] = None,
+        fab: Optional[Fab] = None,
+    ) -> None:
+        """Register new run context for this node."""
+        if run_id not in self.run_infos:
+            initial_run_config = {}
+            if app_dir:
+                # Load from app directory
+                app_path = Path(app_dir)
+                if app_path.is_dir():
+                    override_config = run.override_config if run else {}
+                    initial_run_config = get_fused_config_from_dir(
+                        app_path, override_config
+                    )
+                else:
+                    raise ValueError("The specified `app_dir` must be a directory.")
+            else:
+                if run:
+                    if fab:
+                        # Load pyproject.toml from FAB file and fuse
+                        initial_run_config = get_fused_config_from_fab(fab.content, run)
+                    else:
+                        # Load pyproject.toml from installed FAB and fuse
+                        initial_run_config = get_fused_config(run, flwr_path)
+                else:
+                    initial_run_config = {}
+            self.run_infos[run_id] = RunInfo(
+                initial_run_config=initial_run_config,
+                context=Context(
+                    run_id=run_id,
+                    node_id=self.node_id,
+                    node_config=self.node_config,
+                    state=RecordSet(),
+                    run_config=initial_run_config.copy(),
+                ),
+            )
+
+    def retrieve_context(self, run_id: int) -> Context:
+        """Get run context given a run_id."""
+        if run_id in self.run_infos:
+            return self.run_infos[run_id].context
+
+        raise RuntimeError(
+            f"Context for run_id={run_id} doesn't exist."
+            " A run context must be registered before it can be retrieved or updated "
+            " by a client."
+        )
+
+    def update_context(self, run_id: int, context: Context) -> None:
+        """Update run context."""
+        if context.run_config != self.run_infos[run_id].initial_run_config:
+            raise ValueError(
+                "The `run_config` field of the `Context` object cannot be "
+                f"modified (run_id: {run_id})."
+            )
+        self.run_infos[run_id].context = context

flwr/client/supernode/__init__.py

@@ -0,0 +1,24 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower SuperNode."""
+
+
+from .app import run_client_app as run_client_app
+from .app import run_supernode as run_supernode
+
+__all__ = [
+    "run_client_app",
+    "run_supernode",
+]

flwr/client/supernode/app.py

@@ -0,0 +1,321 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower SuperNode."""
+
+
+import argparse
+import sys
+from logging import DEBUG, ERROR, INFO, WARN
+from pathlib import Path
+from typing import Optional
+
+from cryptography.exceptions import UnsupportedAlgorithm
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives.serialization import (
+    load_ssh_private_key,
+    load_ssh_public_key,
+)
+
+from flwr.common import EventType, event
+from flwr.common.args import try_obtain_root_certificates
+from flwr.common.config import parse_config_args
+from flwr.common.constant import (
+    CLIENTAPPIO_API_DEFAULT_SERVER_ADDRESS,
+    FLEET_API_GRPC_RERE_DEFAULT_ADDRESS,
+    ISOLATION_MODE_PROCESS,
+    ISOLATION_MODE_SUBPROCESS,
+    TRANSPORT_TYPE_GRPC_ADAPTER,
+    TRANSPORT_TYPE_GRPC_RERE,
+    TRANSPORT_TYPE_REST,
+)
+from flwr.common.exit_handlers import register_exit_handlers
+from flwr.common.logger import log, warn_deprecated_feature
+
+from ..app import start_client_internal
+from ..clientapp.utils import get_load_client_app_fn
+
+
+def run_supernode() -> None:
+    """Run Flower SuperNode."""
+    args = _parse_args_run_supernode().parse_args()
+    _warn_deprecated_server_arg(args)
+
+    log(INFO, "Starting Flower SuperNode")
+
+    event(EventType.RUN_SUPERNODE_ENTER)
+
+    # Check if both `--flwr-dir` and `--isolation` were set
+    if args.flwr_dir is not None and args.isolation is not None:
+        log(
+            WARN,
+            "Both `--flwr-dir` and `--isolation` were specified. "
+            "Ignoring `--flwr-dir`.",
+        )
+
+    # Exit if unsupported argument is passed by the user
+    if args.app is not None:
+        log(
+            ERROR,
+            "The `app` argument is deprecated. The SuperNode now automatically "
+            "uses the ClientApp delivered from the SuperLink. Providing the app "
+            "directory manually is no longer supported. Please remove the `app` "
+            "argument from your command.",
+        )
+        sys.exit(1)
+
+    root_certificates = try_obtain_root_certificates(args, args.superlink)
+    load_fn = get_load_client_app_fn(
+        default_app_ref="",
+        app_path=None,
+        flwr_dir=args.flwr_dir,
+        multi_app=True,
+    )
+    authentication_keys = _try_setup_client_authentication(args)
+
+    log(DEBUG, "Isolation mode: %s", args.isolation)
+
+    start_client_internal(
+        server_address=args.superlink,
+        load_client_app_fn=load_fn,
+        transport=args.transport,
+        root_certificates=root_certificates,
+        insecure=args.insecure,
+        authentication_keys=authentication_keys,
+        max_retries=args.max_retries,
+        max_wait_time=args.max_wait_time,
+        node_config=parse_config_args(
+            [args.node_config] if args.node_config else args.node_config
+        ),
+        flwr_path=args.flwr_dir,
+        isolation=args.isolation,
+        clientappio_api_address=args.clientappio_api_address,
+    )
+
+    # Graceful shutdown
+    register_exit_handlers(
+        event_type=EventType.RUN_SUPERNODE_LEAVE,
+    )
+
+
+def run_client_app() -> None:
+    """Run Flower client app."""
+    event(EventType.RUN_CLIENT_APP_ENTER)
+    log(
+        ERROR,
+        "The command `flower-client-app` has been replaced by `flwr run`.",
+    )
+    register_exit_handlers(event_type=EventType.RUN_CLIENT_APP_LEAVE)
+
+
+def _warn_deprecated_server_arg(args: argparse.Namespace) -> None:
+    """Warn about the deprecated argument `--server`."""
+    if args.server != FLEET_API_GRPC_RERE_DEFAULT_ADDRESS:
+        warn = "Passing flag --server is deprecated. Use --superlink instead."
+        warn_deprecated_feature(warn)
+
+        if args.superlink != FLEET_API_GRPC_RERE_DEFAULT_ADDRESS:
+            # if `--superlink` also passed, then
+            # warn user that this argument overrides what was passed with `--server`
+            log(
+                WARN,
+                "Both `--server` and `--superlink` were passed. "
+                "`--server` will be ignored. Connecting to the Superlink Fleet API "
+                "at %s.",
+                args.superlink,
+            )
+        else:
+            args.superlink = args.server
+
+
+def _parse_args_run_supernode() -> argparse.ArgumentParser:
+    """Parse flower-supernode command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Start a Flower SuperNode",
+    )
+
+    parser.add_argument(
+        "app",
+        nargs="?",
+        default=None,
+        help=(
+            "(REMOVED) This argument is removed. The SuperNode now automatically "
+            "uses the ClientApp delivered from the SuperLink, so there is no need to "
+            "provide the app directory manually. This argument will be removed in a "
+            "future version."
+        ),
+    )
+    _parse_args_common(parser)
+    parser.add_argument(
+        "--flwr-dir",
+        default=None,
+        help="""The path containing installed Flower Apps.
+        The default directory is:
+
+        - `$FLWR_HOME/` if `$FLWR_HOME` is defined
+        - `$XDG_DATA_HOME/.flwr/` if `$XDG_DATA_HOME` is defined
+        - `$HOME/.flwr/` in all other cases
+        """,
+    )
+    parser.add_argument(
+        "--isolation",
+        default=ISOLATION_MODE_SUBPROCESS,
+        required=False,
+        choices=[
+            ISOLATION_MODE_SUBPROCESS,
+            ISOLATION_MODE_PROCESS,
+        ],
+        help="Isolation mode when running a `ClientApp` (`subprocess` by default, "
+        "possible values: `subprocess`, `process`). Use `subprocess` to configure "
+        "SuperNode to run a `ClientApp` in a subprocess. Use `process` to indicate "
+        "that a separate independent process gets created outside of SuperNode.",
+    )
+    parser.add_argument(
+        "--clientappio-api-address",
+        default=CLIENTAPPIO_API_DEFAULT_SERVER_ADDRESS,
+        help="ClientAppIo API (gRPC) server address (IPv4, IPv6, or a domain name). "
+        f"By default, it is set to {CLIENTAPPIO_API_DEFAULT_SERVER_ADDRESS}.",
+    )
+
+    return parser
+
+
+def _parse_args_common(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--insecure",
+        action="store_true",
+        help="Run the client without HTTPS. By default, the client runs with "
+        "HTTPS enabled. Use this flag only if you understand the risks.",
+    )
+    ex_group = parser.add_mutually_exclusive_group()
+    ex_group.add_argument(
+        "--grpc-rere",
+        action="store_const",
+        dest="transport",
+        const=TRANSPORT_TYPE_GRPC_RERE,
+        default=TRANSPORT_TYPE_GRPC_RERE,
+        help="Use grpc-rere as a transport layer for the client.",
+    )
+    ex_group.add_argument(
+        "--grpc-adapter",
+        action="store_const",
+        dest="transport",
+        const=TRANSPORT_TYPE_GRPC_ADAPTER,
+        help="Use grpc-adapter as a transport layer for the client.",
+    )
+    ex_group.add_argument(
+        "--rest",
+        action="store_const",
+        dest="transport",
+        const=TRANSPORT_TYPE_REST,
+        help="Use REST as a transport layer for the client.",
+    )
+    parser.add_argument(
+        "--root-certificates",
+        metavar="ROOT_CERT",
+        type=str,
+        help="Specifies the path to the PEM-encoded root certificate file for "
+        "establishing secure HTTPS connections.",
+    )
+    parser.add_argument(
+        "--server",
+        default=FLEET_API_GRPC_RERE_DEFAULT_ADDRESS,
+        help="Server address",
+    )
+    parser.add_argument(
+        "--superlink",
+        default=FLEET_API_GRPC_RERE_DEFAULT_ADDRESS,
+        help="SuperLink Fleet API (gRPC-rere) address (IPv4, IPv6, or a domain name)",
+    )
+    parser.add_argument(
+        "--max-retries",
+        type=int,
+        default=None,
+        help="The maximum number of times the client will try to reconnect to the"
+        "SuperLink before giving up in case of a connection error. By default,"
+        "it is set to None, meaning there is no limit to the number of tries.",
+    )
+    parser.add_argument(
+        "--max-wait-time",
+        type=float,
+        default=None,
+        help="The maximum duration before the client stops trying to"
+        "connect to the SuperLink in case of connection error. By default, it"
+        "is set to None, meaning there is no limit to the total time.",
+    )
+    parser.add_argument(
+        "--auth-supernode-private-key",
+        type=str,
+        help="The SuperNode's private key (as a path str) to enable authentication.",
+    )
+    parser.add_argument(
+        "--auth-supernode-public-key",
+        type=str,
+        help="The SuperNode's public key (as a path str) to enable authentication.",
+    )
+    parser.add_argument(
+        "--node-config",
+        type=str,
+        help="A space separated list of key/value pairs (separated by `=`) to "
+        "configure the SuperNode. "
+        "E.g. --node-config 'key1=\"value1\" partition-id=0 num-partitions=100'",
+    )
+
+
+def _try_setup_client_authentication(
+    args: argparse.Namespace,
+) -> Optional[tuple[ec.EllipticCurvePrivateKey, ec.EllipticCurvePublicKey]]:
+    if not args.auth_supernode_private_key and not args.auth_supernode_public_key:
+        return None
+
+    if not args.auth_supernode_private_key or not args.auth_supernode_public_key:
+        sys.exit(
+            "Authentication requires file paths to both "
+            "'--auth-supernode-private-key' and '--auth-supernode-public-key'"
+            "to be provided (providing only one of them is not sufficient)."
+        )
+
+    try:
+        ssh_private_key = load_ssh_private_key(
+            Path(args.auth_supernode_private_key).read_bytes(),
+            None,
+        )
+        if not isinstance(ssh_private_key, ec.EllipticCurvePrivateKey):
+            raise ValueError()
+    except (ValueError, UnsupportedAlgorithm):
+        sys.exit(
+            "Error: Unable to parse the private key file in "
+            "'--auth-supernode-private-key'. Authentication requires elliptic "
+            "curve private and public key pair. Please ensure that the file "
+            "path points to a valid private key file and try again."
+        )
+
+    try:
+        ssh_public_key = load_ssh_public_key(
+            Path(args.auth_supernode_public_key).read_bytes()
+        )
+        if not isinstance(ssh_public_key, ec.EllipticCurvePublicKey):
+            raise ValueError()
+    except (ValueError, UnsupportedAlgorithm):
+        sys.exit(
+            "Error: Unable to parse the public key file in "
+            "'--auth-supernode-public-key'. Authentication requires elliptic "
+            "curve private and public key pair. Please ensure that the file "
+            "path points to a valid public key file and try again."
+        )
+
+    return (
+        ssh_private_key,
+        ssh_public_key,
+    )

flwr/client/typing.py

@@ -23,6 +23,7 @@ from .client import Client as Client
 
 # Compatibility
 ClientFn = Callable[[str], Client]
+ClientFnExt = Callable[[Context], Client]
 
 ClientAppCallable = Callable[[Message, Context], Message]
 Mod = Callable[[Message, Context, ClientAppCallable], Message]

flwr/common/__init__.py

@@ -22,6 +22,7 @@ from .date import now as now
 from .grpc import GRPC_MAX_MESSAGE_LENGTH
 from .logger import configure as configure
 from .logger import log as log
+from .message import DEFAULT_TTL
 from .message import Error as Error
 from .message import Message as Message
 from .message import Metadata as Metadata
@@ -40,6 +41,7 @@ from .telemetry import event as event
 from .typing import ClientMessage as ClientMessage
 from .typing import Code as Code
 from .typing import Config as Config
+from .typing import ConfigsRecordValues as ConfigsRecordValues
 from .typing import DisconnectRes as DisconnectRes
 from .typing import EvaluateIns as EvaluateIns
 from .typing import EvaluateRes as EvaluateRes
@@ -51,6 +53,7 @@ from .typing import GetPropertiesIns as GetPropertiesIns
 from .typing import GetPropertiesRes as GetPropertiesRes
 from .typing import Metrics as Metrics
 from .typing import MetricsAggregationFn as MetricsAggregationFn
+from .typing import MetricsRecordValues as MetricsRecordValues
 from .typing import NDArray as NDArray
 from .typing import NDArrays as NDArrays
 from .typing import Parameters as Parameters
@@ -62,28 +65,25 @@ from .typing import Status as Status
 
 __all__ = [
     "Array",
-    "array_from_numpy",
-    "bytes_to_ndarray",
     "ClientMessage",
     "Code",
     "Config",
     "ConfigsRecord",
-    "configure",
+    "ConfigsRecordValues",
     "Context",
+    "DEFAULT_TTL",
     "DisconnectRes",
+    "Error",
     "EvaluateIns",
     "EvaluateRes",
-    "event",
     "EventType",
     "FitIns",
     "FitRes",
-    "Error",
+    "GRPC_MAX_MESSAGE_LENGTH",
     "GetParametersIns",
     "GetParametersRes",
     "GetPropertiesIns",
     "GetPropertiesRes",
-    "GRPC_MAX_MESSAGE_LENGTH",
-    "log",
     "Message",
     "MessageType",
     "MessageTypeLegacy",
@@ -91,13 +91,10 @@ __all__ = [
     "Metrics",
     "MetricsAggregationFn",
     "MetricsRecord",
-    "ndarray_to_bytes",
-    "now",
+    "MetricsRecordValues",
     "NDArray",
     "NDArrays",
-    "ndarrays_to_parameters",
     "Parameters",
-    "parameters_to_ndarrays",
     "ParametersRecord",
     "Properties",
     "ReconnectIns",
@@ -105,4 +102,13 @@ __all__ = [
     "Scalar",
     "ServerMessage",
     "Status",
+    "array_from_numpy",
+    "bytes_to_ndarray",
+    "configure",
+    "event",
+    "log",
+    "ndarray_to_bytes",
+    "ndarrays_to_parameters",
+    "now",
+    "parameters_to_ndarrays",
 ]

flwr/common/address.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Flower Labs GmbH. All Rights Reserved.
+# Copyright 2023 Flower Labs GmbH. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,13 +14,15 @@
 # ==============================================================================
 """Flower IP address utils."""
 
+
+import socket
 from ipaddress import ip_address
-from typing import Optional, Tuple
+from typing import Optional
 
 IPV6: int = 6
 
 
-def parse_address(address: str) -> Optional[Tuple[str, int, Optional[bool]]]:
+def parse_address(address: str) -> Optional[tuple[str, int, Optional[bool]]]:
     """Parse an IP address into host, port, and version.
 
     Parameters
@@ -57,3 +59,45 @@ def parse_address(address: str) -> Optional[Tuple[str, int, Optional[bool]]]:
 
     except ValueError:
         return None
+
+
+def is_port_in_use(address: str) -> bool:
+    """Check if the port specified in address is in use.
+
+    Parameters
+    ----------
+    address : str
+        The string representation of a domain, an IPv4, or an IPV6 address
+        with the port number.
+
+        For example, '127.0.0.1:8080', or `[::1]:8080`.
+
+    Returns
+    -------
+    bool
+        If the port provided is in use or can't be parsed,
+        the function will return True, otherwise it will return False.
+    """
+    parsed_address = parse_address(address)
+    if not parsed_address:
+        return True
+    host, port, is_v6 = parsed_address
+
+    if is_v6:
+        protocol = socket.AF_INET6
+    else:
+        protocol = socket.AF_INET
+
+    with socket.socket(protocol, socket.SOCK_STREAM) as s:
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        try:
+            if is_v6:
+                # For IPv6, provide `flowinfo` and `scopeid` as 0
+                s.bind((host, port, 0, 0))
+            else:
+                # For IPv4
+                s.bind((host, port))
+        except OSError:
+            return True
+
+        return False