flwr-nightly 1.10.0.dev20240612__py3-none-any.whl → 1.10.0.dev20240619__py3-none-any.whl

flwr/cli/build.py

@@ -33,7 +33,7 @@ def build(
         Optional[Path],
         typer.Option(help="The Flower project directory to bundle into a FAB"),
     ] = None,
-) -> None:
+) -> str:
     """Build a Flower project into a Flower App Bundle (FAB).
 
     You can run `flwr build` without any argument to bundle the current directory:
@@ -125,6 +125,8 @@ def build(
         f"🎊 Successfully built {fab_filename}.", fg=typer.colors.GREEN, bold=True
     )
 
+    return fab_filename
+
 
 def _load_gitignore(directory: Path) -> pathspec.PathSpec:
     """Load and parse .gitignore file, returning a pathspec."""

flwr/cli/config_utils.py

@@ -14,14 +14,56 @@
 # ==============================================================================
 """Utility to validate the `pyproject.toml` file."""
 
+import zipfile
+from io import BytesIO
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
+from typing import IO, Any, Dict, List, Optional, Tuple, Union
 
 import tomli
 
 from flwr.common import object_ref
 
 
+def get_fab_metadata(fab_file: Union[Path, bytes]) -> Tuple[str, str]:
+    """Extract the fab_id and the fab_version from a FAB file or path.
+
+    Parameters
+    ----------
+    fab_file : Union[Path, bytes]
+        The Flower App Bundle file to validate and extract the metadata from.
+        It can either be a path to the file or the file itself as bytes.
+
+    Returns
+    -------
+    Tuple[str, str]
+        The `fab_version` and `fab_id` of the given Flower App Bundle.
+    """
+    fab_file_archive: Union[Path, IO[bytes]]
+    if isinstance(fab_file, bytes):
+        fab_file_archive = BytesIO(fab_file)
+    elif isinstance(fab_file, Path):
+        fab_file_archive = fab_file
+    else:
+        raise ValueError("fab_file must be either a Path or bytes")
+
+    with zipfile.ZipFile(fab_file_archive, "r") as zipf:
+        with zipf.open("pyproject.toml") as file:
+            toml_content = file.read().decode("utf-8")
+
+        conf = load_from_string(toml_content)
+        if conf is None:
+            raise ValueError("Invalid TOML content in pyproject.toml")
+
+        is_valid, errors, _ = validate(conf, check_module=False)
+        if not is_valid:
+            raise ValueError(errors)
+
+        return (
+            conf["project"]["version"],
+            f"{conf['flower']['publisher']}/{conf['project']['name']}",
+        )
+
+
 def load_and_validate(
     path: Optional[Path] = None,
     check_module: bool = True,
@@ -63,8 +105,7 @@ def load(path: Optional[Path] = None) -> Optional[Dict[str, Any]]:
         return None
 
     with toml_path.open(encoding="utf-8") as toml_file:
-        data = tomli.loads(toml_file.read())
-        return data
+        return load_from_string(toml_file.read())
 
 
 # pylint: disable=too-many-branches
@@ -128,3 +169,12 @@ def validate(
         return False, [reason], []
 
     return True, [], []
+
+
+def load_from_string(toml_content: str) -> Optional[Dict[str, Any]]:
+    """Load TOML content from a string and return as dict."""
+    try:
+        data = tomli.loads(toml_content)
+        return data
+    except tomli.TOMLDecodeError:
+        return None

flwr/cli/install.py

@@ -15,16 +15,18 @@
 """Flower command line interface `install` command."""
 
 
-import os
 import shutil
 import tempfile
 import zipfile
+from io import BytesIO
 from pathlib import Path
-from typing import Optional
+from typing import IO, Optional, Union
 
 import typer
 from typing_extensions import Annotated
 
+from flwr.common.config import get_flwr_dir
+
 from .config_utils import load_and_validate
 from .utils import get_sha256_hash
 
@@ -80,11 +82,24 @@ def install(
 
 
 def install_from_fab(
-    fab_file: Path, flwr_dir: Optional[Path], skip_prompt: bool = False
-) -> None:
+    fab_file: Union[Path, bytes],
+    flwr_dir: Optional[Path],
+    skip_prompt: bool = False,
+) -> Path:
     """Install from a FAB file after extracting and validating."""
+    fab_file_archive: Union[Path, IO[bytes]]
+    fab_name: Optional[str]
+    if isinstance(fab_file, bytes):
+        fab_file_archive = BytesIO(fab_file)
+        fab_name = None
+    elif isinstance(fab_file, Path):
+        fab_file_archive = fab_file
+        fab_name = fab_file.stem
+    else:
+        raise ValueError("fab_file must be either a Path or bytes")
+
     with tempfile.TemporaryDirectory() as tmpdir:
-        with zipfile.ZipFile(fab_file, "r") as zipf:
+        with zipfile.ZipFile(fab_file_archive, "r") as zipf:
             zipf.extractall(tmpdir)
             tmpdir_path = Path(tmpdir)
             info_dir = tmpdir_path / ".info"
@@ -110,15 +125,19 @@ def install_from_fab(
 
             shutil.rmtree(info_dir)
 
-            validate_and_install(tmpdir_path, fab_file.stem, flwr_dir, skip_prompt)
+            installed_path = validate_and_install(
+                tmpdir_path, fab_name, flwr_dir, skip_prompt
+            )
+
+    return installed_path
 
 
 def validate_and_install(
     project_dir: Path,
-    fab_name: str,
+    fab_name: Optional[str],
     flwr_dir: Optional[Path],
     skip_prompt: bool = False,
-) -> None:
+) -> Path:
     """Validate TOML files and install the project to the desired directory."""
     config, _, _ = load_and_validate(project_dir / "pyproject.toml", check_module=False)
 
@@ -134,7 +153,10 @@ def validate_and_install(
     project_name = config["project"]["name"]
     version = config["project"]["version"]
 
-    if fab_name != f"{publisher}.{project_name}.{version.replace('.', '-')}":
+    if (
+        fab_name
+        and fab_name != f"{publisher}.{project_name}.{version.replace('.', '-')}"
+    ):
         typer.secho(
             "❌ FAB file has incorrect name. The file name must follow the format "
             "`<publisher>.<project_name>.<version>.fab`.",
@@ -144,16 +166,7 @@ def validate_and_install(
         raise typer.Exit(code=1)
 
     install_dir: Path = (
-        (
-            Path(
-                os.getenv(
-                    "FLWR_HOME",
-                    f"{os.getenv('XDG_DATA_HOME', os.getenv('HOME'))}/.flwr",
-                )
-            )
-            if not flwr_dir
-            else flwr_dir
-        )
+        (get_flwr_dir() if not flwr_dir else flwr_dir)
         / "apps"
         / publisher
         / project_name
@@ -168,7 +181,7 @@ def validate_and_install(
                 bold=True,
             )
         ):
-            return
+            return install_dir
 
     install_dir.mkdir(parents=True, exist_ok=True)
 
@@ -185,6 +198,8 @@ def validate_and_install(
         bold=True,
     )
 
+    return install_dir
+
 
 def _verify_hashes(list_content: str, tmpdir: Path) -> bool:
     """Verify file hashes based on the LIST content."""

flwr/cli/run/run.py

@@ -16,12 +16,18 @@
 
 import sys
 from enum import Enum
+from logging import DEBUG
 from typing import Optional
 
 import typer
 from typing_extensions import Annotated
 
 from flwr.cli import config_utils
+from flwr.common.constant import SUPEREXEC_DEFAULT_ADDRESS
+from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel
+from flwr.common.logger import log
+from flwr.proto.exec_pb2 import StartRunRequest  # pylint: disable=E0611
+from flwr.proto.exec_pb2_grpc import ExecStub
 from flwr.simulation.run_simulation import _run_simulation
 
 
@@ -31,20 +37,32 @@ class Engine(str, Enum):
     SIMULATION = "simulation"
 
 
+# pylint: disable-next=too-many-locals
 def run(
     engine: Annotated[
         Optional[Engine],
-        typer.Option(case_sensitive=False, help="The ML framework to use"),
+        typer.Option(case_sensitive=False, help="The execution engine to run the app"),
     ] = None,
+    use_superexec: Annotated[
+        bool,
+        typer.Option(
+            case_sensitive=False, help="Use this flag to use the new SuperExec API"
+        ),
+    ] = False,
 ) -> None:
     """Run Flower project."""
+    if use_superexec:
+        _start_superexec_run()
+        return
+
     typer.secho("Loading project configuration... ", fg=typer.colors.BLUE)
 
     config, errors, warnings = config_utils.load_and_validate()
 
     if config is None:
         typer.secho(
-            "Project configuration could not be loaded.\npyproject.toml is invalid:\n"
+            "Project configuration could not be loaded.\n"
+            "pyproject.toml is invalid:\n"
             + "\n".join([f"- {line}" for line in errors]),
             fg=typer.colors.RED,
             bold=True,
@@ -82,3 +100,22 @@ def run(
             fg=typer.colors.RED,
             bold=True,
         )
+
+
+def _start_superexec_run() -> None:
+    def on_channel_state_change(channel_connectivity: str) -> None:
+        """Log channel connectivity."""
+        log(DEBUG, channel_connectivity)
+
+    channel = create_channel(
+        server_address=SUPEREXEC_DEFAULT_ADDRESS,
+        insecure=True,
+        root_certificates=None,
+        max_message_length=GRPC_MAX_MESSAGE_LENGTH,
+        interceptors=None,
+    )
+    channel.subscribe(on_channel_state_change)
+    stub = ExecStub(channel)
+
+    req = StartRunRequest()
+    stub.StartRun(req)

flwr/client/__init__.py

@@ -28,8 +28,8 @@ __all__ = [
     "Client",
     "ClientApp",
     "ClientFn",
-    "mod",
     "NumPyClient",
+    "mod",
     "run_client_app",
     "run_supernode",
     "start_client",

flwr/client/app.py

@@ -19,7 +19,7 @@ import sys
 import time
 from dataclasses import dataclass
 from logging import DEBUG, ERROR, INFO, WARN
-from typing import Callable, ContextManager, Optional, Tuple, Type, Union
+from typing import Callable, ContextManager, Dict, Optional, Tuple, Type, Union
 
 from cryptography.hazmat.primitives.asymmetric import ec
 from grpc import RpcError
@@ -31,6 +31,7 @@ from flwr.common import GRPC_MAX_MESSAGE_LENGTH, EventType, Message, event
 from flwr.common.address import parse_address
 from flwr.common.constant import (
     MISSING_EXTRA_REST,
+    TRANSPORT_TYPE_GRPC_ADAPTER,
     TRANSPORT_TYPE_GRPC_BIDI,
     TRANSPORT_TYPE_GRPC_RERE,
     TRANSPORT_TYPE_REST,
@@ -41,6 +42,7 @@ from flwr.common.logger import log, warn_deprecated_feature
 from flwr.common.message import Error
 from flwr.common.retry_invoker import RetryInvoker, RetryState, exponential
 
+from .grpc_adapter_client.connection import grpc_adapter
 from .grpc_client.connection import grpc_connection
 from .grpc_rere_client.connection import grpc_request_response
 from .message_handler.message_handler import handle_control_message
@@ -177,7 +179,7 @@ def start_client(
 def _start_client_internal(
     *,
     server_address: str,
-    load_client_app_fn: Optional[Callable[[], ClientApp]] = None,
+    load_client_app_fn: Optional[Callable[[str, str], ClientApp]] = None,
     client_fn: Optional[ClientFn] = None,
     client: Optional[Client] = None,
     grpc_max_message_length: int = GRPC_MAX_MESSAGE_LENGTH,
@@ -252,7 +254,7 @@ def _start_client_internal(
 
             client_fn = single_client_factory
 
-        def _load_client_app() -> ClientApp:
+        def _load_client_app(_1: str, _2: str) -> ClientApp:
             return ClientApp(client_fn=client_fn)
 
         load_client_app_fn = _load_client_app
@@ -308,6 +310,8 @@ def _start_client_internal(
     )
 
     node_state = NodeState()
+    # run_id -> (fab_id, fab_version)
+    run_info: Dict[int, Tuple[str, str]] = {}
 
     while not app_state_tracker.interrupt:
         sleep_duration: int = 0
@@ -319,7 +323,6 @@ def _start_client_internal(
             root_certificates,
             authentication_keys,
         ) as conn:
-            # pylint: disable-next=W0612
             receive, send, create_node, delete_node, get_run = conn
 
             # Register node
@@ -356,13 +359,20 @@ def _start_client_internal(
                         send(out_message)
                         break
 
+                    # Get run info
+                    run_id = message.metadata.run_id
+                    if run_id not in run_info:
+                        if get_run is not None:
+                            run_info[run_id] = get_run(run_id)
+                        # If get_run is None, i.e., in grpc-bidi mode
+                        else:
+                            run_info[run_id] = ("", "")
+
                     # Register context for this run
-                    node_state.register_context(run_id=message.metadata.run_id)
+                    node_state.register_context(run_id=run_id)
 
                     # Retrieve context for this run
-                    context = node_state.retrieve_context(
-                        run_id=message.metadata.run_id
-                    )
+                    context = node_state.retrieve_context(run_id=run_id)
 
                     # Create an error reply message that will never be used to prevent
                     # the used-before-assignment linting error
@@ -373,7 +383,7 @@ def _start_client_internal(
                     # Handle app loading and task message
                     try:
                         # Load ClientApp instance
-                        client_app: ClientApp = load_client_app_fn()
+                        client_app: ClientApp = load_client_app_fn(*run_info[run_id])
 
                         # Execute ClientApp
                         reply_message = client_app(message=message, context=context)
@@ -411,7 +421,7 @@ def _start_client_internal(
                     else:
                         # No exception, update node state
                         node_state.update_context(
-                            run_id=message.metadata.run_id,
+                            run_id=run_id,
                             context=context,
                         )
 
@@ -592,6 +602,8 @@ def _init_connection(transport: Optional[str], server_address: str) -> Tuple[
         connection, error_type = http_request_response, RequestsConnectionError
     elif transport == TRANSPORT_TYPE_GRPC_RERE:
         connection, error_type = grpc_request_response, RpcError
+    elif transport == TRANSPORT_TYPE_GRPC_ADAPTER:
+        connection, error_type = grpc_adapter, RpcError
     elif transport == TRANSPORT_TYPE_GRPC_BIDI:
         connection, error_type = grpc_connection, RpcError
     else:

flwr/client/grpc_adapter_client/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Client-side part of the GrpcAdapter transport layer."""

flwr/client/grpc_adapter_client/connection.py

@@ -0,0 +1,94 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Contextmanager for a GrpcAdapter channel to the Flower server."""
+
+
+from contextlib import contextmanager
+from logging import ERROR
+from typing import Callable, Iterator, Optional, Tuple, Union
+
+from cryptography.hazmat.primitives.asymmetric import ec
+
+from flwr.client.grpc_rere_client.connection import grpc_request_response
+from flwr.client.grpc_rere_client.grpc_adapter import GrpcAdapter
+from flwr.common import GRPC_MAX_MESSAGE_LENGTH
+from flwr.common.logger import log
+from flwr.common.message import Message
+from flwr.common.retry_invoker import RetryInvoker
+
+
+@contextmanager
+def grpc_adapter(  # pylint: disable=R0913
+    server_address: str,
+    insecure: bool,
+    retry_invoker: RetryInvoker,
+    max_message_length: int = GRPC_MAX_MESSAGE_LENGTH,  # pylint: disable=W0613
+    root_certificates: Optional[Union[bytes, str]] = None,
+    authentication_keys: Optional[  # pylint: disable=unused-argument
+        Tuple[ec.EllipticCurvePrivateKey, ec.EllipticCurvePublicKey]
+    ] = None,
+) -> Iterator[
+    Tuple[
+        Callable[[], Optional[Message]],
+        Callable[[Message], None],
+        Optional[Callable[[], None]],
+        Optional[Callable[[], None]],
+        Optional[Callable[[int], Tuple[str, str]]],
+    ]
+]:
+    """Primitives for request/response-based interaction with a server via GrpcAdapter.
+
+    Parameters
+    ----------
+    server_address : str
+        The IPv6 address of the server with `http://` or `https://`.
+        If the Flower server runs on the same machine
+        on port 8080, then `server_address` would be `"http://[::]:8080"`.
+    insecure : bool
+        Starts an insecure gRPC connection when True. Enables HTTPS connection
+        when False, using system certificates if `root_certificates` is None.
+    retry_invoker: RetryInvoker
+        `RetryInvoker` object that will try to reconnect the client to the server
+        after gRPC errors. If None, the client will only try to
+        reconnect once after a failure.
+    max_message_length : int
+        Ignored, only present to preserve API-compatibility.
+    root_certificates : Optional[Union[bytes, str]] (default: None)
+        Path of the root certificate. If provided, a secure
+        connection using the certificates will be established to an SSL-enabled
+        Flower server. Bytes won't work for the REST API.
+    authentication_keys : Optional[Tuple[PrivateKey, PublicKey]] (default: None)
+        Client authentication is not supported for this transport type.
+
+    Returns
+    -------
+    receive : Callable
+    send : Callable
+    create_node : Optional[Callable]
+    delete_node : Optional[Callable]
+    get_run : Optional[Callable]
+    """
+    if authentication_keys is not None:
+        log(ERROR, "Client authentication is not supported for this transport type.")
+    with grpc_request_response(
+        server_address=server_address,
+        insecure=insecure,
+        retry_invoker=retry_invoker,
+        max_message_length=max_message_length,
+        root_certificates=root_certificates,
+        authentication_keys=None,  # Authentication is not supported
+        adapter_cls=GrpcAdapter,
+    ) as conn:
+        yield conn

flwr/client/grpc_client/connection.py

@@ -17,7 +17,7 @@
 
 import uuid
 from contextlib import contextmanager
-from logging import DEBUG
+from logging import DEBUG, ERROR
 from pathlib import Path
 from queue import Queue
 from typing import Callable, Iterator, Optional, Tuple, Union, cast
@@ -101,6 +101,8 @@ def grpc_connection(  # pylint: disable=R0913, R0915
         The PEM-encoded root certificates as a byte string or a path string.
         If provided, a secure connection using the certificates will be
         established to an SSL-enabled Flower server.
+    authentication_keys : Optional[Tuple[PrivateKey, PublicKey]] (default: None)
+        Client authentication is not supported for this transport type.
 
     Returns
     -------
@@ -123,6 +125,8 @@ def grpc_connection(  # pylint: disable=R0913, R0915
     """
     if isinstance(root_certificates, str):
         root_certificates = Path(root_certificates).read_bytes()
+    if authentication_keys is not None:
+        log(ERROR, "Client authentication is not supported for this transport type.")
 
     channel = create_channel(
         server_address=server_address,

flwr/client/grpc_rere_client/connection.py

@@ -55,6 +55,7 @@ from flwr.proto.run_pb2 import GetRunRequest, GetRunResponse  # pylint: disable=
 from flwr.proto.task_pb2 import TaskIns  # pylint: disable=E0611
 
 from .client_interceptor import AuthenticateClientInterceptor
+from .grpc_adapter import GrpcAdapter
 
 
 def on_channel_state_change(channel_connectivity: str) -> None:
@@ -72,7 +73,7 @@ def grpc_request_response(  # pylint: disable=R0913, R0914, R0915
     authentication_keys: Optional[
         Tuple[ec.EllipticCurvePrivateKey, ec.EllipticCurvePublicKey]
     ] = None,
-    adapter_cls: Optional[Type[FleetStub]] = None,
+    adapter_cls: Optional[Union[Type[FleetStub], Type[GrpcAdapter]]] = None,
 ) -> Iterator[
     Tuple[
         Callable[[], Optional[Message]],
@@ -106,6 +107,11 @@ def grpc_request_response(  # pylint: disable=R0913, R0914, R0915
         Path of the root certificate. If provided, a secure
         connection using the certificates will be established to an SSL-enabled
         Flower server. Bytes won't work for the REST API.
+    authentication_keys : Optional[Tuple[PrivateKey, PublicKey]] (default: None)
+        Tuple containing the elliptic curve private key and public key for
+        authentication from the cryptography library.
+        Source: https://cryptography.io/en/latest/hazmat/primitives/asymmetric/ec/
+        Used to establish an authenticated connection with the server.
 
     Returns
     -------
@@ -113,6 +119,7 @@ def grpc_request_response(  # pylint: disable=R0913, R0914, R0915
     send : Callable
     create_node : Optional[Callable]
     delete_node : Optional[Callable]
+    get_run : Optional[Callable]
     """
     if isinstance(root_certificates, str):
         root_certificates = Path(root_certificates).read_bytes()