flwr-nightly 1.13.0.dev20241021__py3-none-any.whl → 1.13.0.dev20241111__py3-none-any.whl

flwr/simulation/run_simulation.py

@@ -21,7 +21,6 @@ import logging
 import sys
 import threading
 import traceback
-from argparse import Namespace
 from logging import DEBUG, ERROR, INFO, WARNING
 from pathlib import Path
 from time import sleep
@@ -29,69 +28,28 @@ from typing import Any, Optional
 
 from flwr.cli.config_utils import load_and_validate
 from flwr.client import ClientApp
-from flwr.common import EventType, event, log
+from flwr.common import Context, EventType, RecordSet, event, log, now
 from flwr.common.config import get_fused_config_from_dir, parse_config_args
-from flwr.common.constant import RUN_ID_NUM_BYTES
+from flwr.common.constant import RUN_ID_NUM_BYTES, Status
 from flwr.common.logger import (
     set_logger_propagation,
     update_console_handler,
-    warn_deprecated_feature,
     warn_deprecated_feature_with_example,
 )
-from flwr.common.typing import Run, UserConfig
+from flwr.common.typing import Run, RunStatus, UserConfig
 from flwr.server.driver import Driver, InMemoryDriver
-from flwr.server.run_serverapp import run as run_server_app
+from flwr.server.run_serverapp import run as _run
 from flwr.server.server_app import ServerApp
 from flwr.server.superlink.fleet import vce
 from flwr.server.superlink.fleet.vce.backend.backend import BackendConfig
-from flwr.server.superlink.state import StateFactory
-from flwr.server.superlink.state.utils import generate_rand_int_from_bytes
+from flwr.server.superlink.linkstate import LinkStateFactory
+from flwr.server.superlink.linkstate.in_memory_linkstate import RunRecord
+from flwr.server.superlink.linkstate.utils import generate_rand_int_from_bytes
 from flwr.simulation.ray_transport.utils import (
     enable_tf_gpu_growth as enable_gpu_growth,
 )
 
 
-def _check_args_do_not_interfere(args: Namespace) -> bool:
-    """Ensure decoupling of flags for different ways to start the simulation."""
-    mode_one_args = ["app", "run_config"]
-    mode_two_args = ["client_app", "server_app"]
-
-    def _resolve_message(conflict_keys: list[str]) -> str:
-        return ",".join([f"`--{key}`".replace("_", "-") for key in conflict_keys])
-
-    # When passing `--app`, `--app-dir` is ignored
-    if args.app and args.app_dir:
-        log(ERROR, "Either `--app` or `--app-dir` can be set, but not both.")
-        return False
-
-    if any(getattr(args, key) for key in mode_one_args):
-        if any(getattr(args, key) for key in mode_two_args):
-            log(
-                ERROR,
-                "Passing any of {%s} alongside with any of {%s}",
-                _resolve_message(mode_one_args),
-                _resolve_message(mode_two_args),
-            )
-            return False
-
-        if not args.app:
-            log(ERROR, "You need to pass --app")
-            return False
-
-        return True
-
-    # Ensure all args are set (required for the non-FAB mode of execution)
-    if not all(getattr(args, key) for key in mode_two_args):
-        log(
-            ERROR,
-            "Passing all of %s keys are required.",
-            _resolve_message(mode_two_args),
-        )
-        return False
-
-    return True
-
-
 def _replace_keys(d: Any, match: str, target: str) -> Any:
     if isinstance(d, dict):
         return {
@@ -114,19 +72,6 @@ def run_simulation_from_cli() -> None:
         event_details={"backend": args.backend, "num-supernodes": args.num_supernodes},
     )
 
-    # Add warnings for deprecated server_app and client_app arguments
-    if args.server_app:
-        warn_deprecated_feature(
-            "The `--server-app` argument is deprecated. "
-            "Please use the `--app` argument instead."
-        )
-
-    if args.client_app:
-        warn_deprecated_feature(
-            "The `--client-app` argument is deprecated. "
-            "Use the `--app` argument instead."
-        )
-
     if args.enable_tf_gpu_growth:
         warn_deprecated_feature_with_example(
             "Passing `--enable-tf-gpu-growth` is deprecated.",
@@ -143,60 +88,39 @@ def run_simulation_from_cli() -> None:
         backend_config_dict = _replace_keys(backend_config_dict, match="-", target="_")
         log(DEBUG, "backend_config_dict: %s", backend_config_dict)
 
-    # We are supporting two modes for the CLI entrypoint:
-    # 1) Running an app dir containing a `pyproject.toml`
-    # 2) Running any ClientApp and SeverApp w/o pyproject.toml being present
-    # For 2), some CLI args are compulsory, but they are not required for 1)
-    # We first do these checks
-    args_check_pass = _check_args_do_not_interfere(args)
-    if not args_check_pass:
-        sys.exit("Simulation Engine cannot start.")
-
     run_id = (
         generate_rand_int_from_bytes(RUN_ID_NUM_BYTES)
         if args.run_id is None
         else args.run_id
     )
-    if args.app:
-        # Mode 1
-        app_path = Path(args.app)
-        if not app_path.is_dir():
-            log(ERROR, "--app is not a directory")
-            sys.exit("Simulation Engine cannot start.")
-
-        # Load pyproject.toml
-        config, errors, warnings = load_and_validate(
-            app_path / "pyproject.toml", check_module=False
-        )
-        if errors:
-            raise ValueError(errors)
 
-        if warnings:
-            log(WARNING, warnings)
+    app_path = Path(args.app)
+    if not app_path.is_dir():
+        log(ERROR, "--app is not a directory")
+        sys.exit("Simulation Engine cannot start.")
 
-        if config is None:
-            raise ValueError("Config extracted from FAB's pyproject.toml is not valid")
+    # Load pyproject.toml
+    config, errors, warnings = load_and_validate(
+        app_path / "pyproject.toml", check_module=False
+    )
+    if errors:
+        raise ValueError(errors)
 
-        # Get ClientApp and SeverApp components
-        app_components = config["tool"]["flwr"]["app"]["components"]
-        client_app_attr = app_components["clientapp"]
-        server_app_attr = app_components["serverapp"]
+    if warnings:
+        log(WARNING, warnings)
 
-        override_config = parse_config_args(
-            [args.run_config] if args.run_config else args.run_config
-        )
-        fused_config = get_fused_config_from_dir(app_path, override_config)
-        app_dir = args.app
-        is_app = True
+    if config is None:
+        raise ValueError("Config extracted from FAB's pyproject.toml is not valid")
 
-    else:
-        # Mode 2
-        client_app_attr = args.client_app
-        server_app_attr = args.server_app
-        override_config = {}
-        fused_config = None
-        app_dir = args.app_dir
-        is_app = False
+    # Get ClientApp and SeverApp components
+    app_components = config["tool"]["flwr"]["app"]["components"]
+    client_app_attr = app_components["clientapp"]
+    server_app_attr = app_components["serverapp"]
+
+    override_config = parse_config_args(
+        [args.run_config] if args.run_config else args.run_config
+    )
+    fused_config = get_fused_config_from_dir(app_path, override_config)
 
     # Create run
     run = Run(
@@ -213,13 +137,13 @@ def run_simulation_from_cli() -> None:
         num_supernodes=args.num_supernodes,
         backend_name=args.backend,
         backend_config=backend_config_dict,
-        app_dir=app_dir,
+        app_dir=args.app,
         run=run,
         enable_tf_gpu_growth=args.enable_tf_gpu_growth,
         delay_start=args.delay_start,
         verbose_logging=args.verbose,
         server_app_run_config=fused_config,
-        is_app=is_app,
+        is_app=True,
         exit_event=EventType.CLI_FLOWER_SIMULATION_LEAVE,
     )
 
@@ -310,6 +234,7 @@ def run_serverapp_th(
     f_stop: threading.Event,
     has_exception: threading.Event,
     enable_tf_gpu_growth: bool,
+    run_id: int,
 ) -> threading.Thread:
     """Run SeverApp in a thread."""
 
@@ -332,11 +257,20 @@ def run_serverapp_th(
                 log(INFO, "Enabling GPU growth for Tensorflow on the server thread.")
                 enable_gpu_growth()
 
+            # Initialize Context
+            context = Context(
+                run_id=run_id,
+                node_id=0,
+                node_config={},
+                state=RecordSet(),
+                run_config=_server_app_run_config,
+            )
+
             # Run ServerApp
-            run_server_app(
+            _run(
                 driver=_driver,
+                context=context,
                 server_app_dir=_server_app_dir,
-                server_app_run_config=_server_app_run_config,
                 server_app_attr=_server_app_attr,
                 loaded_server_app=_server_app,
             )
@@ -389,7 +323,7 @@ def _main_loop(
 ) -> None:
     """Start ServerApp on a separate thread, then launch Simulation Engine."""
     # Initialize StateFactory
-    state_factory = StateFactory(":flwr-in-memory-state:")
+    state_factory = LinkStateFactory(":flwr-in-memory-state:")
 
     f_stop = threading.Event()
     # A Threading event to indicate if an exception was raised in the ServerApp thread
@@ -399,13 +333,21 @@ def _main_loop(
     try:
         # Register run
         log(DEBUG, "Pre-registering run with id %s", run.run_id)
-        state_factory.state().run_ids[run.run_id] = run  # type: ignore
+        init_status = RunStatus(Status.RUNNING, "", "")
+        state_factory.state().run_ids[run.run_id] = RunRecord(  # type: ignore
+            run=run,
+            status=init_status,
+            starting_at=now().isoformat(),
+            running_at=now().isoformat(),
+            finished_at="",
+        )
 
         if server_app_run_config is None:
             server_app_run_config = {}
 
         # Initialize Driver
-        driver = InMemoryDriver(run_id=run.run_id, state_factory=state_factory)
+        driver = InMemoryDriver(state_factory=state_factory)
+        driver.set_run(run_id=run.run_id)
 
         # Get and run ServerApp thread
         serverapp_th = run_serverapp_th(
@@ -417,6 +359,7 @@ def _main_loop(
             f_stop=f_stop,
             has_exception=server_app_thread_has_exception,
             enable_tf_gpu_growth=enable_tf_gpu_growth,
+            run_id=run.run_id,
         )
 
         # Buffer time so the `ServerApp` in separate thread is ready
@@ -566,20 +509,10 @@ def _parse_args_run_simulation() -> argparse.ArgumentParser:
     parser.add_argument(
         "--app",
         type=str,
-        default=None,
+        required=True,
         help="Path to a directory containing a FAB-like structure with a "
         "pyproject.toml.",
     )
-    parser.add_argument(
-        "--server-app",
-        help="(DEPRECATED: use --app instead) For example: `server:app` or "
-        "`project.package.module:wrapper.app`",
-    )
-    parser.add_argument(
-        "--client-app",
-        help="(DEPRECATED: use --app instead) For example: `client:app` or "
-        "`project.package.module:wrapper.app`",
-    )
     parser.add_argument(
         "--num-supernodes",
         type=int,
@@ -628,13 +561,6 @@ def _parse_args_run_simulation() -> argparse.ArgumentParser:
         help="When unset, only INFO, WARNING and ERROR log messages will be shown. "
         "If set, DEBUG-level logs will be displayed. ",
     )
-    parser.add_argument(
-        "--app-dir",
-        default="",
-        help="Add specified directory to the PYTHONPATH and load"
-        "ClientApp and ServerApp from there."
-        " Default: current working directory.",
-    )
     parser.add_argument(
         "--flwr-dir",
         default=None,

flwr/simulation/simulationio_connection.py

@@ -0,0 +1,86 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower SimulationIo connection."""
+
+
+from logging import DEBUG, WARNING
+from typing import Optional, cast
+
+import grpc
+
+from flwr.common.constant import SIMULATIONIO_API_DEFAULT_ADDRESS
+from flwr.common.grpc import create_channel
+from flwr.common.logger import log
+from flwr.proto.simulationio_pb2_grpc import SimulationIoStub  # pylint: disable=E0611
+
+
+class SimulationIoConnection:
+    """`SimulationIoConnection` provides an interface to the SimulationIo API.
+
+    Parameters
+    ----------
+    simulationio_service_address : str (default: "[::]:9094")
+        The address (URL, IPv6, IPv4) of the SuperLink SimulationIo API service.
+    root_certificates : Optional[bytes] (default: None)
+        The PEM-encoded root certificates as a byte string.
+        If provided, a secure connection using the certificates will be
+        established to an SSL-enabled Flower server.
+    """
+
+    def __init__(  # pylint: disable=too-many-arguments
+        self,
+        simulationio_service_address: str = SIMULATIONIO_API_DEFAULT_ADDRESS,
+        root_certificates: Optional[bytes] = None,
+    ) -> None:
+        self._addr = simulationio_service_address
+        self._cert = root_certificates
+        self._grpc_stub: Optional[SimulationIoStub] = None
+        self._channel: Optional[grpc.Channel] = None
+
+    @property
+    def _is_connected(self) -> bool:
+        """Check if connected to the SimulationIo API server."""
+        return self._channel is not None
+
+    @property
+    def _stub(self) -> SimulationIoStub:
+        """SimulationIo stub."""
+        if not self._is_connected:
+            self._connect()
+        return cast(SimulationIoStub, self._grpc_stub)
+
+    def _connect(self) -> None:
+        """Connect to the SimulationIo API."""
+        if self._is_connected:
+            log(WARNING, "Already connected")
+            return
+        self._channel = create_channel(
+            server_address=self._addr,
+            insecure=(self._cert is None),
+            root_certificates=self._cert,
+        )
+        self._grpc_stub = SimulationIoStub(self._channel)
+        log(DEBUG, "[SimulationIO] Connected to %s", self._addr)
+
+    def _disconnect(self) -> None:
+        """Disconnect from the SimulationIo API."""
+        if not self._is_connected:
+            log(DEBUG, "Already disconnected")
+            return
+        channel: grpc.Channel = self._channel
+        self._channel = None
+        self._grpc_stub = None
+        channel.close()
+        log(DEBUG, "[SimulationIO] Disconnected")

flwr/superexec/app.py

@@ -16,20 +16,11 @@
 
 import argparse
 import sys
-from logging import INFO, WARN
-from pathlib import Path
-from typing import Optional
+from logging import INFO
 
-import grpc
-
-from flwr.common import EventType, event, log
-from flwr.common.address import parse_address
-from flwr.common.config import parse_config_args
-from flwr.common.constant import EXEC_API_DEFAULT_ADDRESS
-from flwr.common.exit_handlers import register_exit_handlers
+from flwr.common import log
 from flwr.common.object_ref import load_app, validate
 
-from .exec_grpc import run_superexec_api_grpc
 from .executor import Executor
 
 
@@ -37,133 +28,14 @@ def run_superexec() -> None:
     """Run Flower SuperExec."""
     log(INFO, "Starting Flower SuperExec")
 
-    event(EventType.RUN_SUPEREXEC_ENTER)
-
-    args = _parse_args_run_superexec().parse_args()
-
-    # Parse IP address
-    parsed_address = parse_address(args.address)
-    if not parsed_address:
-        sys.exit(f"SuperExec IP address ({args.address}) cannot be parsed.")
-    host, port, is_v6 = parsed_address
-    address = f"[{host}]:{port}" if is_v6 else f"{host}:{port}"
-
-    # Obtain certificates
-    certificates = _try_obtain_certificates(args)
-
-    # Start SuperExec API
-    superexec_server: grpc.Server = run_superexec_api_grpc(
-        address=address,
-        executor=_load_executor(args),
-        certificates=certificates,
-        config=parse_config_args(
-            [args.executor_config] if args.executor_config else args.executor_config
-        ),
-    )
-
-    grpc_servers = [superexec_server]
-
-    # Graceful shutdown
-    register_exit_handlers(
-        event_type=EventType.RUN_SUPEREXEC_LEAVE,
-        grpc_servers=grpc_servers,
-        bckg_threads=None,
-    )
-
-    superexec_server.wait_for_termination()
-
-
-def _parse_args_run_superexec() -> argparse.ArgumentParser:
-    """Parse command line arguments for SuperExec."""
-    parser = argparse.ArgumentParser(
-        description="Start a Flower SuperExec",
-    )
-    parser.add_argument(
-        "--address",
-        help="SuperExec (gRPC) server address (IPv4, IPv6, or a domain name)",
-        default=EXEC_API_DEFAULT_ADDRESS,
-    )
-    parser.add_argument(
-        "--executor",
-        help="For example: `deployment:exec` or `project.package.module:wrapper.exec`.",
-        default="flwr.superexec.deployment:executor",
-    )
-    parser.add_argument(
-        "--executor-dir",
-        help="The directory for the executor.",
-        default=".",
-    )
-    parser.add_argument(
-        "--executor-config",
-        help="Key-value pairs for the executor config, separated by spaces. "
-        'For example:\n\n`--executor-config \'superlink="superlink:9091" '
-        'root-certificates="certificates/superlink-ca.crt"\'`',
-    )
-    parser.add_argument(
-        "--insecure",
-        action="store_true",
-        help="Run the SuperExec without HTTPS, regardless of whether certificate "
-        "paths are provided. By default, the server runs with HTTPS enabled. "
-        "Use this flag only if you understand the risks.",
-    )
-    parser.add_argument(
-        "--ssl-certfile",
-        help="SuperExec server SSL certificate file (as a path str) "
-        "to create a secure connection.",
-        type=str,
-        default=None,
-    )
-    parser.add_argument(
-        "--ssl-keyfile",
-        help="SuperExec server SSL private key file (as a path str) "
-        "to create a secure connection.",
-        type=str,
-    )
-    parser.add_argument(
-        "--ssl-ca-certfile",
-        help="SuperExec server SSL CA certificate file (as a path str) "
-        "to create a secure connection.",
-        type=str,
-    )
-    return parser
-
-
-def _try_obtain_certificates(
-    args: argparse.Namespace,
-) -> Optional[tuple[bytes, bytes, bytes]]:
-    # Obtain certificates
-    if args.insecure:
-        log(WARN, "Option `--insecure` was set. Starting insecure HTTP server.")
-        return None
-    # Check if certificates are provided
-    if args.ssl_certfile and args.ssl_keyfile and args.ssl_ca_certfile:
-        if not Path(args.ssl_ca_certfile).is_file():
-            sys.exit("Path argument `--ssl-ca-certfile` does not point to a file.")
-        if not Path(args.ssl_certfile).is_file():
-            sys.exit("Path argument `--ssl-certfile` does not point to a file.")
-        if not Path(args.ssl_keyfile).is_file():
-            sys.exit("Path argument `--ssl-keyfile` does not point to a file.")
-        certificates = (
-            Path(args.ssl_ca_certfile).read_bytes(),  # CA certificate
-            Path(args.ssl_certfile).read_bytes(),  # server certificate
-            Path(args.ssl_keyfile).read_bytes(),  # server private key
-        )
-        return certificates
-    if args.ssl_certfile or args.ssl_keyfile or args.ssl_ca_certfile:
-        sys.exit(
-            "You need to provide valid file paths to `--ssl-certfile`, "
-            "`--ssl-keyfile`, and `—-ssl-ca-certfile` to create a secure "
-            "connection in SuperExec server (gRPC-rere)."
-        )
     sys.exit(
-        "Certificates are required unless running in insecure mode. "
-        "Please provide certificate paths to `--ssl-certfile`, "
-        "`--ssl-keyfile`, and `—-ssl-ca-certfile` or run the server "
-        "in insecure mode using '--insecure' if you understand the risks."
+        "Manually launching the SuperExec is deprecated. Since `flwr 1.13.0` "
+        "the executor service runs in the SuperLink. Launching it manually is not "
+        "recommended."
     )
 
 
-def _load_executor(
+def load_executor(
     args: argparse.Namespace,
 ) -> Executor:
     """Get the executor plugin."""