flwr-nightly 1.10.0.dev20240612__py3-none-any.whl → 1.10.0.dev20240624__py3-none-any.whl

flwr/superexec/app.py

@@ -0,0 +1,178 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower SuperExec app."""
+
+import argparse
+import sys
+from logging import INFO, WARN
+from pathlib import Path
+from typing import Optional, Tuple
+
+import grpc
+
+from flwr.common import EventType, event, log
+from flwr.common.address import parse_address
+from flwr.common.constant import SUPEREXEC_DEFAULT_ADDRESS
+from flwr.common.exit_handlers import register_exit_handlers
+from flwr.common.object_ref import load_app, validate
+
+from .exec_grpc import run_superexec_api_grpc
+from .executor import Executor
+
+
+def run_superexec() -> None:
+    """Run Flower SuperExec."""
+    log(INFO, "Starting Flower SuperExec")
+
+    event(EventType.RUN_SUPEREXEC_ENTER)
+
+    args = _parse_args_run_superexec().parse_args()
+
+    # Parse IP address
+    parsed_address = parse_address(args.address)
+    if not parsed_address:
+        sys.exit(f"SuperExec IP address ({args.address}) cannot be parsed.")
+    host, port, is_v6 = parsed_address
+    address = f"[{host}]:{port}" if is_v6 else f"{host}:{port}"
+
+    # Obtain certificates
+    certificates = _try_obtain_certificates(args)
+
+    # Start SuperExec API
+    superexec_server: grpc.Server = run_superexec_api_grpc(
+        address=address,
+        executor=_load_executor(args),
+        certificates=certificates,
+    )
+
+    grpc_servers = [superexec_server]
+
+    # Graceful shutdown
+    register_exit_handlers(
+        event_type=EventType.RUN_SUPEREXEC_LEAVE,
+        grpc_servers=grpc_servers,
+        bckg_threads=None,
+    )
+
+    superexec_server.wait_for_termination()
+
+
+def _parse_args_run_superexec() -> argparse.ArgumentParser:
+    """Parse command line arguments for SuperExec."""
+    parser = argparse.ArgumentParser(
+        description="Start a Flower SuperExec",
+    )
+    parser.add_argument(
+        "executor",
+        help="For example: `deployment:exec` or `project.package.module:wrapper.exec`.",
+    )
+    parser.add_argument(
+        "--address",
+        help="SuperExec (gRPC) server address (IPv4, IPv6, or a domain name)",
+        default=SUPEREXEC_DEFAULT_ADDRESS,
+    )
+    parser.add_argument(
+        "--executor-dir",
+        help="The directory for the executor.",
+        default=".",
+    )
+    parser.add_argument(
+        "--insecure",
+        action="store_true",
+        help="Run the SuperExec without HTTPS, regardless of whether certificate "
+        "paths are provided. By default, the server runs with HTTPS enabled. "
+        "Use this flag only if you understand the risks.",
+    )
+    parser.add_argument(
+        "--ssl-certfile",
+        help="SuperExec server SSL certificate file (as a path str) "
+        "to create a secure connection.",
+        type=str,
+        default=None,
+    )
+    parser.add_argument(
+        "--ssl-keyfile",
+        help="SuperExec server SSL private key file (as a path str) "
+        "to create a secure connection.",
+        type=str,
+    )
+    parser.add_argument(
+        "--ssl-ca-certfile",
+        help="SuperExec server SSL CA certificate file (as a path str) "
+        "to create a secure connection.",
+        type=str,
+    )
+    return parser
+
+
+def _try_obtain_certificates(
+    args: argparse.Namespace,
+) -> Optional[Tuple[bytes, bytes, bytes]]:
+    # Obtain certificates
+    if args.insecure:
+        log(WARN, "Option `--insecure` was set. Starting insecure HTTP server.")
+        return None
+    # Check if certificates are provided
+    if args.ssl_certfile and args.ssl_keyfile and args.ssl_ca_certfile:
+        if not Path.is_file(args.ssl_ca_certfile):
+            sys.exit("Path argument `--ssl-ca-certfile` does not point to a file.")
+        if not Path.is_file(args.ssl_certfile):
+            sys.exit("Path argument `--ssl-certfile` does not point to a file.")
+        if not Path.is_file(args.ssl_keyfile):
+            sys.exit("Path argument `--ssl-keyfile` does not point to a file.")
+        certificates = (
+            Path(args.ssl_ca_certfile).read_bytes(),  # CA certificate
+            Path(args.ssl_certfile).read_bytes(),  # server certificate
+            Path(args.ssl_keyfile).read_bytes(),  # server private key
+        )
+        return certificates
+    if args.ssl_certfile or args.ssl_keyfile or args.ssl_ca_certfile:
+        sys.exit(
+            "You need to provide valid file paths to `--ssl-certfile`, "
+            "`--ssl-keyfile`, and `—-ssl-ca-certfile` to create a secure "
+            "connection in SuperExec server (gRPC-rere)."
+        )
+    sys.exit(
+        "Certificates are required unless running in insecure mode. "
+        "Please provide certificate paths to `--ssl-certfile`, "
+        "`--ssl-keyfile`, and `—-ssl-ca-certfile` or run the server "
+        "in insecure mode using '--insecure' if you understand the risks."
+    )
+
+
+def _load_executor(
+    args: argparse.Namespace,
+) -> Executor:
+    """Get the executor plugin."""
+    if args.executor_dir is not None:
+        sys.path.insert(0, args.executor_dir)
+
+    executor_ref: str = args.executor
+    valid, error_msg = validate(executor_ref)
+    if not valid and error_msg:
+        raise LoadExecutorError(error_msg) from None
+
+    executor = load_app(executor_ref, LoadExecutorError, args.executor_dir)
+
+    if not isinstance(executor, Executor):
+        raise LoadExecutorError(
+            f"Attribute {executor_ref} is not of type {Executor}",
+        ) from None
+
+    return executor
+
+
+class LoadExecutorError(Exception):
+    """Error when trying to load `Executor`."""

flwr/superexec/exec_grpc.py

@@ -0,0 +1,51 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""SuperExec gRPC API."""
+
+from logging import INFO
+from typing import Optional, Tuple
+
+import grpc
+
+from flwr.common import GRPC_MAX_MESSAGE_LENGTH
+from flwr.common.logger import log
+from flwr.proto.exec_pb2_grpc import add_ExecServicer_to_server
+from flwr.server.superlink.fleet.grpc_bidi.grpc_server import generic_create_grpc_server
+
+from .exec_servicer import ExecServicer
+from .executor import Executor
+
+
+def run_superexec_api_grpc(
+    address: str,
+    executor: Executor,
+    certificates: Optional[Tuple[bytes, bytes, bytes]],
+) -> grpc.Server:
+    """Run SuperExec API (gRPC, request-response)."""
+    exec_servicer: grpc.Server = ExecServicer(
+        executor=executor,
+    )
+    superexec_add_servicer_to_server_fn = add_ExecServicer_to_server
+    superexec_grpc_server = generic_create_grpc_server(
+        servicer_and_add_fn=(exec_servicer, superexec_add_servicer_to_server_fn),
+        server_address=address,
+        max_message_length=GRPC_MAX_MESSAGE_LENGTH,
+        certificates=certificates,
+    )
+
+    log(INFO, "Flower ECE: Starting SuperExec API (gRPC-rere) on %s", address)
+    superexec_grpc_server.start()
+
+    return superexec_grpc_server

flwr/superexec/exec_servicer.py

@@ -0,0 +1,65 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""SuperExec API servicer."""
+
+
+from logging import ERROR, INFO
+from typing import Any, Dict, Generator
+
+import grpc
+
+from flwr.common.logger import log
+from flwr.proto import exec_pb2_grpc  # pylint: disable=E0611
+from flwr.proto.exec_pb2 import (  # pylint: disable=E0611
+    StartRunRequest,
+    StartRunResponse,
+    StreamLogsRequest,
+    StreamLogsResponse,
+)
+
+from .executor import Executor, RunTracker
+
+
+class ExecServicer(exec_pb2_grpc.ExecServicer):
+    """SuperExec API servicer."""
+
+    def __init__(self, executor: Executor) -> None:
+        self.executor = executor
+        self.runs: Dict[int, RunTracker] = {}
+
+    def StartRun(
+        self, request: StartRunRequest, context: grpc.ServicerContext
+    ) -> StartRunResponse:
+        """Create run ID."""
+        log(INFO, "ExecServicer.StartRun")
+
+        run = self.executor.start_run(request.fab_file)
+
+        if run is None:
+            log(ERROR, "Executor failed to start run")
+            return StartRunResponse()
+
+        self.runs[run.run_id] = run
+
+        return StartRunResponse(run_id=run.run_id)
+
+    def StreamLogs(
+        self, request: StreamLogsRequest, context: grpc.ServicerContext
+    ) -> Generator[StreamLogsResponse, Any, None]:
+        """Get logs."""
+        logs = ["a", "b", "c"]
+        while context.is_active():
+            for i in range(len(logs)):  # pylint: disable=C0200
+                yield StreamLogsResponse(log_output=logs[i])

flwr/superexec/executor.py

@@ -0,0 +1,54 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Execute and monitor a Flower run."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from subprocess import Popen
+from typing import Optional
+
+
+@dataclass
+class RunTracker:
+    """Track a Flower run (composed of a run_id and the associated process)."""
+
+    run_id: int
+    proc: Popen  # type: ignore
+
+
+class Executor(ABC):
+    """Execute and monitor a Flower run."""
+
+    @abstractmethod
+    def start_run(
+        self,
+        fab_file: bytes,
+    ) -> Optional[RunTracker]:
+        """Start a run using the given Flower FAB ID and version.
+
+        This method creates a new run on the SuperLink, returns its run_id
+        and also starts the run execution.
+
+        Parameters
+        ----------
+        fab_file : bytes
+            The Flower App Bundle file bytes.
+
+        Returns
+        -------
+        run_id : Optional[RunTracker]
+            The run_id and the associated process of the run created by the SuperLink,
+            or `None` if it fails.
+        """

{flwr_nightly-1.10.0.dev20240612.dist-info → flwr_nightly-1.10.0.dev20240624.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.10.0.dev20240612
+Version: 1.10.0.dev20240624
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0
@@ -204,6 +204,7 @@ Other [examples](https://github.com/adap/flower/tree/main/examples):
 - [Flower with KaplanMeierFitter from the lifelines library](https://github.com/adap/flower/tree/main/examples/federated-kaplan-meier-fitter)
 - [Sample Level Privacy with Opacus](https://github.com/adap/flower/tree/main/examples/opacus)
 - [Sample Level Privacy with TensorFlow-Privacy](https://github.com/adap/flower/tree/main/examples/tensorflow-privacy)
+- [Flower with a Tabular Dataset](https://github.com/adap/flower/tree/main/examples/fl-tabular)
 
 ## Community