flwr-nightly 1.10.0.dev20240612__py3-none-any.whl → 1.10.0.dev20240614__py3-none-any.whl

flwr/proto/exec_pb2.pyi

@@ -0,0 +1,32 @@
+"""
+@generated by mypy-protobuf.  Do not edit manually!
+isort:skip_file
+"""
+import builtins
+import google.protobuf.descriptor
+import google.protobuf.message
+import typing_extensions
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+class StartRunRequest(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+    FAB_FILE_FIELD_NUMBER: builtins.int
+    fab_file: builtins.bytes
+    def __init__(self,
+        *,
+        fab_file: builtins.bytes = ...,
+        ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["fab_file",b"fab_file"]) -> None: ...
+global___StartRunRequest = StartRunRequest
+
+class StartRunResponse(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+    RUN_ID_FIELD_NUMBER: builtins.int
+    run_id: builtins.int
+    def __init__(self,
+        *,
+        run_id: builtins.int = ...,
+        ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["run_id",b"run_id"]) -> None: ...
+global___StartRunResponse = StartRunResponse

flwr/proto/exec_pb2_grpc.py

@@ -0,0 +1,67 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+from flwr.proto import exec_pb2 as flwr_dot_proto_dot_exec__pb2
+
+
+class ExecStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.StartRun = channel.unary_unary(
+                '/flwr.proto.Exec/StartRun',
+                request_serializer=flwr_dot_proto_dot_exec__pb2.StartRunRequest.SerializeToString,
+                response_deserializer=flwr_dot_proto_dot_exec__pb2.StartRunResponse.FromString,
+                )
+
+
+class ExecServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def StartRun(self, request, context):
+        """Start run upon request
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_ExecServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'StartRun': grpc.unary_unary_rpc_method_handler(
+                    servicer.StartRun,
+                    request_deserializer=flwr_dot_proto_dot_exec__pb2.StartRunRequest.FromString,
+                    response_serializer=flwr_dot_proto_dot_exec__pb2.StartRunResponse.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'flwr.proto.Exec', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class Exec(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def StartRun(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/flwr.proto.Exec/StartRun',
+            flwr_dot_proto_dot_exec__pb2.StartRunRequest.SerializeToString,
+            flwr_dot_proto_dot_exec__pb2.StartRunResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

flwr/proto/exec_pb2_grpc.pyi

@@ -0,0 +1,27 @@
+"""
+@generated by mypy-protobuf.  Do not edit manually!
+isort:skip_file
+"""
+import abc
+import flwr.proto.exec_pb2
+import grpc
+
+class ExecStub:
+    def __init__(self, channel: grpc.Channel) -> None: ...
+    StartRun: grpc.UnaryUnaryMultiCallable[
+        flwr.proto.exec_pb2.StartRunRequest,
+        flwr.proto.exec_pb2.StartRunResponse]
+    """Start run upon request"""
+
+
+class ExecServicer(metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def StartRun(self,
+        request: flwr.proto.exec_pb2.StartRunRequest,
+        context: grpc.ServicerContext,
+    ) -> flwr.proto.exec_pb2.StartRunResponse:
+        """Start run upon request"""
+        pass
+
+
+def add_ExecServicer_to_server(servicer: ExecServicer, server: grpc.Server) -> None: ...

flwr/server/app.py

@@ -200,15 +200,7 @@ def run_superlink() -> None:
     args = _parse_args_run_superlink().parse_args()
 
     # Parse IP address
-    parsed_driver_address = parse_address(args.driver_api_address)
-    if not parsed_driver_address:
-        sys.exit(f"Driver IP address ({args.driver_api_address}) cannot be parsed.")
-    driver_host, driver_port, driver_is_v6 = parsed_driver_address
-    driver_address = (
-        f"[{driver_host}]:{driver_port}"
-        if driver_is_v6
-        else f"{driver_host}:{driver_port}"
-    )
+    driver_address, _, _ = _format_address(args.driver_api_address)
 
     # Obtain certificates
     certificates = _try_obtain_certificates(args)
@@ -231,13 +223,8 @@ def run_superlink() -> None:
             if args.fleet_api_type == TRANSPORT_TYPE_GRPC_RERE
             else ADDRESS_FLEET_API_REST
         )
-    parsed_fleet_address = parse_address(args.fleet_api_address)
-    if not parsed_fleet_address:
-        sys.exit(f"Fleet IP address ({args.fleet_api_address}) cannot be parsed.")
-    fleet_host, fleet_port, fleet_is_v6 = parsed_fleet_address
-    fleet_address = (
-        f"[{fleet_host}]:{fleet_port}" if fleet_is_v6 else f"{fleet_host}:{fleet_port}"
-    )
+
+    fleet_address, host, port = _format_address(args.fleet_api_address)
 
     num_workers = args.fleet_api_num_workers
     if num_workers != 1:
@@ -267,8 +254,8 @@ def run_superlink() -> None:
         fleet_thread = threading.Thread(
             target=_run_fleet_api_rest,
             args=(
-                fleet_host,
-                fleet_port,
+                host,
+                port,
                 ssl_keyfile,
                 ssl_certfile,
                 state_factory,
@@ -325,6 +312,16 @@ def run_superlink() -> None:
         driver_server.wait_for_termination(timeout=1)
 
 
+def _format_address(address: str) -> Tuple[str, str, int]:
+    parsed_address = parse_address(address)
+    if not parsed_address:
+        sys.exit(
+            f"Address ({address}) cannot be parsed (expected: URL or IPv4 or IPv6)."
+        )
+    host, port, is_v6 = parsed_address
+    return (f"[{host}]:{port}" if is_v6 else f"{host}:{port}", host, port)
+
+
 def _try_setup_client_authentication(
     args: argparse.Namespace,
     certificates: Optional[Tuple[bytes, bytes, bytes]],

flwr/simulation/run_simulation.py

@@ -53,6 +53,7 @@ def run_simulation_from_cli() -> None:
         backend_name=args.backend,
         backend_config=backend_config_dict,
         app_dir=args.app_dir,
+        run_id=args.run_id,
         enable_tf_gpu_growth=args.enable_tf_gpu_growth,
         verbose_logging=args.verbose,
     )
@@ -168,6 +169,13 @@ def run_serverapp_th(
     return serverapp_th
 
 
+def _init_run_id(driver: InMemoryDriver, state: StateFactory, run_id: int) -> None:
+    """Create a run with a given `run_id`."""
+    log(DEBUG, "Pre-registering run with id %s", run_id)
+    state.state().run_ids[run_id] = ("", "")  # type: ignore
+    driver.run_id = run_id
+
+
 # pylint: disable=too-many-locals
 def _main_loop(
     num_supernodes: int,
@@ -175,6 +183,7 @@ def _main_loop(
     backend_config_stream: str,
     app_dir: str,
     enable_tf_gpu_growth: bool,
+    run_id: Optional[int] = None,
     client_app: Optional[ClientApp] = None,
     client_app_attr: Optional[str] = None,
     server_app: Optional[ServerApp] = None,
@@ -195,6 +204,9 @@ def _main_loop(
         # Initialize Driver
         driver = InMemoryDriver(state_factory)
 
+        if run_id:
+            _init_run_id(driver, state_factory, run_id)
+
         # Get and run ServerApp thread
         serverapp_th = run_serverapp_th(
             server_app_attr=server_app_attr,
@@ -244,6 +256,7 @@ def _run_simulation(
     client_app_attr: Optional[str] = None,
     server_app_attr: Optional[str] = None,
     app_dir: str = "",
+    run_id: Optional[int] = None,
     enable_tf_gpu_growth: bool = False,
     verbose_logging: bool = False,
 ) -> None:
@@ -283,6 +296,9 @@ def _run_simulation(
         Add specified directory to the PYTHONPATH and load `ClientApp` from there.
         (Default: current working directory.)
 
+    run_id : Optional[int]
+        An integer specifying the ID of the run started when running this function.
+
     enable_tf_gpu_growth : bool (default: False)
         A boolean to indicate whether to enable GPU growth on the main thread. This is
         desirable if you make use of a TensorFlow model on your `ServerApp` while
@@ -322,6 +338,7 @@ def _run_simulation(
         backend_config_stream,
         app_dir,
         enable_tf_gpu_growth,
+        run_id,
         client_app,
         client_app_attr,
         server_app,
@@ -413,5 +430,10 @@ def _parse_args_run_simulation() -> argparse.ArgumentParser:
         "ClientApp and ServerApp from there."
         " Default: current working directory.",
     )
+    parser.add_argument(
+        "--run-id",
+        type=int,
+        help="Sets the ID of the run started by the Simulation Engine.",
+    )
 
     return parser

flwr/superexec/__init__.py

@@ -0,0 +1,21 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower SuperExec service."""
+
+from .app import run_superexec as run_superexec
+
+__all__ = [
+    "run_superexec",
+]

flwr/superexec/app.py

@@ -0,0 +1,178 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower SuperExec app."""
+
+import argparse
+import sys
+from logging import INFO, WARN
+from pathlib import Path
+from typing import Optional, Tuple
+
+import grpc
+
+from flwr.common import EventType, event, log
+from flwr.common.address import parse_address
+from flwr.common.constant import SUPEREXEC_DEFAULT_ADDRESS
+from flwr.common.exit_handlers import register_exit_handlers
+from flwr.common.object_ref import load_app, validate
+
+from .exec_grpc import run_superexec_api_grpc
+from .executor import Executor
+
+
+def run_superexec() -> None:
+    """Run Flower SuperExec."""
+    log(INFO, "Starting Flower SuperExec")
+
+    event(EventType.RUN_SUPEREXEC_ENTER)
+
+    args = _parse_args_run_superexec().parse_args()
+
+    # Parse IP address
+    parsed_address = parse_address(args.address)
+    if not parsed_address:
+        sys.exit(f"SuperExec IP address ({args.address}) cannot be parsed.")
+    host, port, is_v6 = parsed_address
+    address = f"[{host}]:{port}" if is_v6 else f"{host}:{port}"
+
+    # Obtain certificates
+    certificates = _try_obtain_certificates(args)
+
+    # Start SuperExec API
+    superexec_server: grpc.Server = run_superexec_api_grpc(
+        address=address,
+        executor=_load_executor(args),
+        certificates=certificates,
+    )
+
+    grpc_servers = [superexec_server]
+
+    # Graceful shutdown
+    register_exit_handlers(
+        event_type=EventType.RUN_SUPEREXEC_LEAVE,
+        grpc_servers=grpc_servers,
+        bckg_threads=None,
+    )
+
+    superexec_server.wait_for_termination()
+
+
+def _parse_args_run_superexec() -> argparse.ArgumentParser:
+    """Parse command line arguments for SuperExec."""
+    parser = argparse.ArgumentParser(
+        description="Start a Flower SuperExec",
+    )
+    parser.add_argument(
+        "executor",
+        help="For example: `deployment:exec` or `project.package.module:wrapper.exec`.",
+    )
+    parser.add_argument(
+        "--address",
+        help="SuperExec (gRPC) server address (IPv4, IPv6, or a domain name)",
+        default=SUPEREXEC_DEFAULT_ADDRESS,
+    )
+    parser.add_argument(
+        "--executor-dir",
+        help="The directory for the executor.",
+        default=".",
+    )
+    parser.add_argument(
+        "--insecure",
+        action="store_true",
+        help="Run the SuperExec without HTTPS, regardless of whether certificate "
+        "paths are provided. By default, the server runs with HTTPS enabled. "
+        "Use this flag only if you understand the risks.",
+    )
+    parser.add_argument(
+        "--ssl-certfile",
+        help="SuperExec server SSL certificate file (as a path str) "
+        "to create a secure connection.",
+        type=str,
+        default=None,
+    )
+    parser.add_argument(
+        "--ssl-keyfile",
+        help="SuperExec server SSL private key file (as a path str) "
+        "to create a secure connection.",
+        type=str,
+    )
+    parser.add_argument(
+        "--ssl-ca-certfile",
+        help="SuperExec server SSL CA certificate file (as a path str) "
+        "to create a secure connection.",
+        type=str,
+    )
+    return parser
+
+
+def _try_obtain_certificates(
+    args: argparse.Namespace,
+) -> Optional[Tuple[bytes, bytes, bytes]]:
+    # Obtain certificates
+    if args.insecure:
+        log(WARN, "Option `--insecure` was set. Starting insecure HTTP server.")
+        return None
+    # Check if certificates are provided
+    if args.ssl_certfile and args.ssl_keyfile and args.ssl_ca_certfile:
+        if not Path.is_file(args.ssl_ca_certfile):
+            sys.exit("Path argument `--ssl-ca-certfile` does not point to a file.")
+        if not Path.is_file(args.ssl_certfile):
+            sys.exit("Path argument `--ssl-certfile` does not point to a file.")
+        if not Path.is_file(args.ssl_keyfile):
+            sys.exit("Path argument `--ssl-keyfile` does not point to a file.")
+        certificates = (
+            Path(args.ssl_ca_certfile).read_bytes(),  # CA certificate
+            Path(args.ssl_certfile).read_bytes(),  # server certificate
+            Path(args.ssl_keyfile).read_bytes(),  # server private key
+        )
+        return certificates
+    if args.ssl_certfile or args.ssl_keyfile or args.ssl_ca_certfile:
+        sys.exit(
+            "You need to provide valid file paths to `--ssl-certfile`, "
+            "`--ssl-keyfile`, and `—-ssl-ca-certfile` to create a secure "
+            "connection in SuperExec server (gRPC-rere)."
+        )
+    sys.exit(
+        "Certificates are required unless running in insecure mode. "
+        "Please provide certificate paths to `--ssl-certfile`, "
+        "`--ssl-keyfile`, and `—-ssl-ca-certfile` or run the server "
+        "in insecure mode using '--insecure' if you understand the risks."
+    )
+
+
+def _load_executor(
+    args: argparse.Namespace,
+) -> Executor:
+    """Get the executor plugin."""
+    if args.executor_dir is not None:
+        sys.path.insert(0, args.executor_dir)
+
+    executor_ref: str = args.executor
+    valid, error_msg = validate(executor_ref)
+    if not valid and error_msg:
+        raise LoadExecutorError(error_msg) from None
+
+    executor = load_app(executor_ref, LoadExecutorError)
+
+    if not isinstance(executor, Executor):
+        raise LoadExecutorError(
+            f"Attribute {executor_ref} is not of type {Executor}",
+        ) from None
+
+    return executor
+
+
+class LoadExecutorError(Exception):
+    """Error when trying to load `Executor`."""

flwr/superexec/exec_grpc.py

@@ -0,0 +1,51 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""SuperExec gRPC API."""
+
+from logging import INFO
+from typing import Optional, Tuple
+
+import grpc
+
+from flwr.common import GRPC_MAX_MESSAGE_LENGTH
+from flwr.common.logger import log
+from flwr.proto.exec_pb2_grpc import add_ExecServicer_to_server
+from flwr.server.superlink.fleet.grpc_bidi.grpc_server import generic_create_grpc_server
+
+from .exec_servicer import ExecServicer
+from .executor import Executor
+
+
+def run_superexec_api_grpc(
+    address: str,
+    executor: Executor,
+    certificates: Optional[Tuple[bytes, bytes, bytes]],
+) -> grpc.Server:
+    """Run SuperExec API (gRPC, request-response)."""
+    exec_servicer: grpc.Server = ExecServicer(
+        executor=executor,
+    )
+    superexec_add_servicer_to_server_fn = add_ExecServicer_to_server
+    superexec_grpc_server = generic_create_grpc_server(
+        servicer_and_add_fn=(exec_servicer, superexec_add_servicer_to_server_fn),
+        server_address=address,
+        max_message_length=GRPC_MAX_MESSAGE_LENGTH,
+        certificates=certificates,
+    )
+
+    log(INFO, "Flower ECE: Starting SuperExec API (gRPC-rere) on %s", address)
+    superexec_grpc_server.start()
+
+    return superexec_grpc_server

flwr/superexec/exec_servicer.py

@@ -0,0 +1,54 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""SuperExec API servicer."""
+
+
+from logging import ERROR, INFO
+from typing import Dict
+
+import grpc
+
+from flwr.common.logger import log
+from flwr.proto import exec_pb2_grpc  # pylint: disable=E0611
+from flwr.proto.exec_pb2 import (  # pylint: disable=E0611
+    StartRunRequest,
+    StartRunResponse,
+)
+
+from .executor import Executor, RunTracker
+
+
+class ExecServicer(exec_pb2_grpc.ExecServicer):
+    """SuperExec API servicer."""
+
+    def __init__(self, executor: Executor) -> None:
+        self.executor = executor
+        self.runs: Dict[int, RunTracker] = {}
+
+    def StartRun(
+        self, request: StartRunRequest, context: grpc.ServicerContext
+    ) -> StartRunResponse:
+        """Create run ID."""
+        log(INFO, "ExecServicer.StartRun")
+
+        run = self.executor.start_run(request.fab_file)
+
+        if run is None:
+            log(ERROR, "Executor failed to start run")
+            return StartRunResponse()
+
+        self.runs[run.run_id] = run
+
+        return StartRunResponse(run_id=run.run_id)

flwr/superexec/executor.py

@@ -0,0 +1,54 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Execute and monitor a Flower run."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from subprocess import Popen
+from typing import Optional
+
+
+@dataclass
+class RunTracker:
+    """Track a Flower run (composed of a run_id and the associated process)."""
+
+    run_id: int
+    proc: Popen  # type: ignore
+
+
+class Executor(ABC):
+    """Execute and monitor a Flower run."""
+
+    @abstractmethod
+    def start_run(
+        self,
+        fab_file: bytes,
+    ) -> Optional[RunTracker]:
+        """Start a run using the given Flower FAB ID and version.
+
+        This method creates a new run on the SuperLink, returns its run_id
+        and also starts the run execution.
+
+        Parameters
+        ----------
+        fab_file : bytes
+            The Flower App Bundle file bytes.
+
+        Returns
+        -------
+        run_id : Optional[RunTracker]
+            The run_id and the associated process of the run created by the SuperLink,
+            or `None` if it fails.
+        """

{flwr_nightly-1.10.0.dev20240612.dist-info → flwr_nightly-1.10.0.dev20240614.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.10.0.dev20240612
+Version: 1.10.0.dev20240614
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0