flwr 1.20.0__py3-none-any.whl → 1.21.0__py3-none-any.whl

flwr/simulation/app.py

@@ -16,10 +16,8 @@
 
 
 import argparse
-import gc
 from logging import DEBUG, ERROR, INFO
 from queue import Queue
-from time import sleep
 from typing import Optional
 
 from flwr.cli.config_utils import get_fab_metadata
@@ -36,6 +34,7 @@ from flwr.common.config import (
 )
 from flwr.common.constant import (
     SIMULATIONIO_API_DEFAULT_CLIENT_ADDRESS,
+    ExecPluginType,
     Status,
     SubStatus,
 )
@@ -57,19 +56,23 @@ from flwr.common.serde import (
     run_status_to_proto,
 )
 from flwr.common.typing import RunStatus
+from flwr.proto.appio_pb2 import (  # pylint: disable=E0611
+    PullAppInputsRequest,
+    PullAppInputsResponse,
+    PushAppOutputsRequest,
+)
 from flwr.proto.run_pb2 import (  # pylint: disable=E0611
     GetFederationOptionsRequest,
     GetFederationOptionsResponse,
     UpdateRunStatusRequest,
 )
-from flwr.proto.simulationio_pb2 import (  # pylint: disable=E0611
-    PullSimulationInputsRequest,
-    PullSimulationInputsResponse,
-    PushSimulationOutputsRequest,
-)
+from flwr.proto.simulationio_pb2_grpc import SimulationIoStub
 from flwr.server.superlink.fleet.vce.backend.backend import BackendConfig
 from flwr.simulation.run_simulation import _run_simulation
 from flwr.simulation.simulationio_connection import SimulationIoConnection
+from flwr.supercore.app_utils import start_parent_process_monitor
+from flwr.supercore.superexec.plugin import SimulationExecPlugin
+from flwr.supercore.superexec.run_superexec import run_with_deprecation_warning
 
 
 def flwr_simulation() -> None:
@@ -80,14 +83,27 @@ def flwr_simulation() -> None:
 
     args = _parse_args_run_flwr_simulation().parse_args()
 
-    log(INFO, "Starting Flower Simulation")
-
     if not args.insecure:
         flwr_exit(
             ExitCode.COMMON_TLS_NOT_SUPPORTED,
-            "`flwr-simulation` does not support TLS yet. ",
+            "`flwr-simulation` does not support TLS yet.",
+        )
+
+    # Disallow long-running `flwr-simulation` processes
+    if args.token is None:
+        run_with_deprecation_warning(
+            cmd="flwr-simulation",
+            plugin_type=ExecPluginType.SIMULATION,
+            plugin_class=SimulationExecPlugin,
+            stub_class=SimulationIoStub,
+            appio_api_address=args.simulationio_api_address,
+            flwr_dir=args.flwr_dir,
+            parent_pid=args.parent_pid,
+            warn_run_once=args.run_once,
         )
+        return
 
+    log(INFO, "Starting Flower Simulation")
     log(
         DEBUG,
         "Starting isolated `Simulation` connected to SuperLink SimulationAppIo API "
@@ -97,23 +113,29 @@ def flwr_simulation() -> None:
     run_simulation_process(
         simulationio_api_address=args.simulationio_api_address,
         log_queue=log_queue,
-        run_once=args.run_once,
+        token=args.token,
         flwr_dir_=args.flwr_dir,
         certificates=None,
+        parent_pid=args.parent_pid,
     )
 
     # Restore stdout/stderr
     restore_output()
 
 
-def run_simulation_process(  # pylint: disable=R0914, disable=W0212, disable=R0915
+def run_simulation_process(  # pylint: disable=R0913, R0914, R0915, R0917, W0212
     simulationio_api_address: str,
     log_queue: Queue[Optional[str]],
-    run_once: bool,
+    token: str,
     flwr_dir_: Optional[str] = None,
     certificates: Optional[bytes] = None,
+    parent_pid: Optional[int] = None,
 ) -> None:
     """Run Flower Simulation process."""
+    # Start monitoring the parent process if a PID is provided
+    if parent_pid is not None:
+        start_parent_process_monitor(parent_pid)
+
     conn = SimulationIoConnection(
         simulationio_service_address=simulationio_api_address,
         root_certificates=certificates,
@@ -123,165 +145,146 @@ def run_simulation_process(  # pylint: disable=R0914, disable=W0212, disable=R09
     flwr_dir = get_flwr_dir(flwr_dir_)
     log_uploader = None
     heartbeat_sender = None
+    run_status = None
+
+    try:
+        # Pull SimulationInputs from LinkState
+        req = PullAppInputsRequest(token=token)
+        res: PullAppInputsResponse = conn._stub.PullAppInputs(req)
+        context = context_from_proto(res.context)
+        run = run_from_proto(res.run)
+        fab = fab_from_proto(res.fab)
+
+        # Start log uploader for this run
+        log_uploader = start_log_uploader(
+            log_queue=log_queue,
+            node_id=context.node_id,
+            run_id=run.run_id,
+            stub=conn._stub,
+        )
 
-    while True:
+        log(DEBUG, "Simulation process starts FAB installation.")
+        install_from_fab(fab.content, flwr_dir=flwr_dir, skip_prompt=True)
 
-        try:
-            # Pull SimulationInputs from LinkState
-            req = PullSimulationInputsRequest()
-            res: PullSimulationInputsResponse = conn._stub.PullSimulationInputs(req)
-            if not res.HasField("run"):
-                sleep(3)
-                run_status = None
-                continue
-
-            context = context_from_proto(res.context)
-            run = run_from_proto(res.run)
-            fab = fab_from_proto(res.fab)
-
-            # Start log uploader for this run
-            log_uploader = start_log_uploader(
-                log_queue=log_queue,
-                node_id=context.node_id,
-                run_id=run.run_id,
-                stub=conn._stub,
-            )
+        fab_id, fab_version = get_fab_metadata(fab.content)
 
-            log(DEBUG, "Simulation process starts FAB installation.")
-            install_from_fab(fab.content, flwr_dir=flwr_dir, skip_prompt=True)
+        app_path = get_project_dir(fab_id, fab_version, fab.hash_str, flwr_dir)
+        config = get_project_config(app_path)
 
-            fab_id, fab_version = get_fab_metadata(fab.content)
+        # Get ClientApp and SeverApp components
+        app_components = config["tool"]["flwr"]["app"]["components"]
+        client_app_attr = app_components["clientapp"]
+        server_app_attr = app_components["serverapp"]
+        fused_config = get_fused_config_from_dir(app_path, run.override_config)
 
-            app_path = get_project_dir(fab_id, fab_version, fab.hash_str, flwr_dir)
-            config = get_project_config(app_path)
+        # Update run_config in context
+        context.run_config = fused_config
 
-            # Get ClientApp and SeverApp components
-            app_components = config["tool"]["flwr"]["app"]["components"]
-            client_app_attr = app_components["clientapp"]
-            server_app_attr = app_components["serverapp"]
-            fused_config = get_fused_config_from_dir(app_path, run.override_config)
+        log(
+            DEBUG,
+            "Flower will load ServerApp `%s` in %s",
+            server_app_attr,
+            app_path,
+        )
+        log(
+            DEBUG,
+            "Flower will load ClientApp `%s` in %s",
+            client_app_attr,
+            app_path,
+        )
 
-            # Update run_config in context
-            context.run_config = fused_config
+        # Change status to Running
+        run_status_proto = run_status_to_proto(RunStatus(Status.RUNNING, "", ""))
+        conn._stub.UpdateRunStatus(
+            UpdateRunStatusRequest(run_id=run.run_id, run_status=run_status_proto)
+        )
 
-            log(
-                DEBUG,
-                "Flower will load ServerApp `%s` in %s",
-                server_app_attr,
-                app_path,
-            )
-            log(
-                DEBUG,
-                "Flower will load ClientApp `%s` in %s",
-                client_app_attr,
-                app_path,
-            )
+        # Pull Federation Options
+        fed_opt_res: GetFederationOptionsResponse = conn._stub.GetFederationOptions(
+            GetFederationOptionsRequest(run_id=run.run_id)
+        )
+        federation_options = config_record_from_proto(fed_opt_res.federation_options)
+
+        # Unflatten underlying dict
+        fed_opt = unflatten_dict({**federation_options})
+
+        # Extract configs values of interest
+        num_supernodes = fed_opt.get("num-supernodes")
+        if num_supernodes is None:
+            raise ValueError("Federation options expects `num-supernodes` to be set.")
+        backend_config: BackendConfig = fed_opt.get("backend", {})
+        verbose: bool = fed_opt.get("verbose", False)
+        enable_tf_gpu_growth: bool = fed_opt.get("enable_tf_gpu_growth", False)
+
+        event(
+            EventType.FLWR_SIMULATION_RUN_ENTER,
+            event_details={
+                "backend": "ray",
+                "num-supernodes": num_supernodes,
+                "run-id-hash": get_sha256_hash(run.run_id),
+            },
+        )
 
-            # Change status to Running
-            run_status_proto = run_status_to_proto(RunStatus(Status.RUNNING, "", ""))
-            conn._stub.UpdateRunStatus(
-                UpdateRunStatusRequest(run_id=run.run_id, run_status=run_status_proto)
-            )
+        # Set up heartbeat sender
+        heartbeat_fn = get_grpc_app_heartbeat_fn(
+            conn._stub,
+            run.run_id,
+            failure_message="Heartbeat failed unexpectedly. The SuperLink could "
+            "not find the provided run ID, or the run status is invalid.",
+        )
+        heartbeat_sender = HeartbeatSender(heartbeat_fn)
+        heartbeat_sender.start()
+
+        # Launch the simulation
+        updated_context = _run_simulation(
+            server_app_attr=server_app_attr,
+            client_app_attr=client_app_attr,
+            num_supernodes=num_supernodes,
+            backend_config=backend_config,
+            app_dir=str(app_path),
+            run=run,
+            enable_tf_gpu_growth=enable_tf_gpu_growth,
+            verbose_logging=verbose,
+            server_app_run_config=fused_config,
+            is_app=True,
+            exit_event=EventType.FLWR_SIMULATION_RUN_LEAVE,
+        )
 
-            # Pull Federation Options
-            fed_opt_res: GetFederationOptionsResponse = conn._stub.GetFederationOptions(
-                GetFederationOptionsRequest(run_id=run.run_id)
-            )
-            federation_options = config_record_from_proto(
-                fed_opt_res.federation_options
-            )
+        # Send resulting context
+        context_proto = context_to_proto(updated_context)
+        out_req = PushAppOutputsRequest(
+            token=token, run_id=run.run_id, context=context_proto
+        )
+        _ = conn._stub.PushAppOutputs(out_req)
 
-            # Unflatten underlying dict
-            fed_opt = unflatten_dict({**federation_options})
-
-            # Extract configs values of interest
-            num_supernodes = fed_opt.get("num-supernodes")
-            if num_supernodes is None:
-                raise ValueError(
-                    "Federation options expects `num-supernodes` to be set."
-                )
-            backend_config: BackendConfig = fed_opt.get("backend", {})
-            verbose: bool = fed_opt.get("verbose", False)
-            enable_tf_gpu_growth: bool = fed_opt.get("enable_tf_gpu_growth", False)
-
-            event(
-                EventType.FLWR_SIMULATION_RUN_ENTER,
-                event_details={
-                    "backend": "ray",
-                    "num-supernodes": num_supernodes,
-                    "run-id-hash": get_sha256_hash(run.run_id),
-                },
-            )
+        run_status = RunStatus(Status.FINISHED, SubStatus.COMPLETED, "")
 
-            # Set up heartbeat sender
-            heartbeat_fn = get_grpc_app_heartbeat_fn(
-                conn._stub,
-                run.run_id,
-                failure_message="Heartbeat failed unexpectedly. The SuperLink could "
-                "not find the provided run ID, or the run status is invalid.",
-            )
-            heartbeat_sender = HeartbeatSender(heartbeat_fn)
-            heartbeat_sender.start()
-
-            # Launch the simulation
-            updated_context = _run_simulation(
-                server_app_attr=server_app_attr,
-                client_app_attr=client_app_attr,
-                num_supernodes=num_supernodes,
-                backend_config=backend_config,
-                app_dir=str(app_path),
-                run=run,
-                enable_tf_gpu_growth=enable_tf_gpu_growth,
-                verbose_logging=verbose,
-                server_app_run_config=fused_config,
-                is_app=True,
-                exit_event=EventType.FLWR_SIMULATION_RUN_LEAVE,
-            )
+    except Exception as ex:  # pylint: disable=broad-exception-caught
+        exc_entity = "Simulation"
+        log(ERROR, "%s raised an exception", exc_entity, exc_info=ex)
+        run_status = RunStatus(Status.FINISHED, SubStatus.FAILED, str(ex))
+
+    finally:
+        # Stop heartbeat sender
+        if heartbeat_sender:
+            heartbeat_sender.stop()
+
+        # Stop log uploader for this run and upload final logs
+        if log_uploader:
+            stop_log_uploader(log_queue, log_uploader)
 
-            # Send resulting context
-            context_proto = context_to_proto(updated_context)
-            out_req = PushSimulationOutputsRequest(
-                run_id=run.run_id, context=context_proto
+        # Update run status
+        if run_status:
+            run_status_proto = run_status_to_proto(run_status)
+            conn._stub.UpdateRunStatus(
+                UpdateRunStatusRequest(run_id=run.run_id, run_status=run_status_proto)
             )
-            _ = conn._stub.PushSimulationOutputs(out_req)
-
-            run_status = RunStatus(Status.FINISHED, SubStatus.COMPLETED, "")
-
-        except Exception as ex:  # pylint: disable=broad-exception-caught
-            exc_entity = "Simulation"
-            log(ERROR, "%s raised an exception", exc_entity, exc_info=ex)
-            run_status = RunStatus(Status.FINISHED, SubStatus.FAILED, str(ex))
-
-        finally:
-            # Stop heartbeat sender
-            if heartbeat_sender:
-                heartbeat_sender.stop()
-                heartbeat_sender = None
-
-            # Stop log uploader for this run and upload final logs
-            if log_uploader:
-                stop_log_uploader(log_queue, log_uploader)
-                log_uploader = None
-
-            # Update run status
-            if run_status:
-                run_status_proto = run_status_to_proto(run_status)
-                conn._stub.UpdateRunStatus(
-                    UpdateRunStatusRequest(
-                        run_id=run.run_id, run_status=run_status_proto
-                    )
-                )
-
-            # Clean up the Context if it exists
-            try:
-                del updated_context
-            except NameError:
-                pass
-            gc.collect()
-
-        # Stop the loop if `flwr-simulation` is expected to process a single run
-        if run_once:
-            break
+
+        # Clean up the Context if it exists
+        try:
+            del updated_context
+        except NameError:
+            pass
 
 
 def _parse_args_run_flwr_simulation() -> argparse.ArgumentParser:
@@ -296,11 +299,5 @@ def _parse_args_run_flwr_simulation() -> argparse.ArgumentParser:
         help="Address of SuperLink's SimulationIO API (IPv4, IPv6, or a domain name)."
         f"By default, it is set to {SIMULATIONIO_API_DEFAULT_CLIENT_ADDRESS}.",
     )
-    parser.add_argument(
-        "--run-once",
-        action="store_true",
-        help="When set, this process will start a single simulation "
-        "for a pending Run. If no pending run the process will exit. ",
-    )
     add_args_flwr_app_common(parser=parser)
     return parser

flwr/supercore/app_utils.py

@@ -0,0 +1,58 @@
+# Copyright 2025 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utility functions for app processes."""
+
+
+import os
+import signal
+import threading
+import time
+
+if os.name == "nt":
+    from ctypes import windll  # type: ignore
+
+
+def _pid_exists(pid: int) -> bool:
+    """Check if a process with the given PID exists.
+
+    This works on Unix-like systems and Windows.
+    """
+    # Use `ctypes` to check if the process exists on Windows
+    if os.name == "nt":
+        handle = windll.kernel32.OpenProcess(0x1000, False, pid)
+        if handle:
+            windll.kernel32.CloseHandle(handle)
+            return True
+        return False
+    # Use `os.kill` on Unix-like systems
+    try:
+        os.kill(pid, 0)
+    except OSError:
+        return False
+    return True
+
+
+def start_parent_process_monitor(
+    parent_pid: int,
+) -> None:
+    """Monitor the parent process and exit if it terminates."""
+
+    def monitor() -> None:
+        while True:
+            time.sleep(0.2)
+            if not _pid_exists(parent_pid):
+                os.kill(os.getpid(), signal.SIGKILL)
+
+    threading.Thread(target=monitor, daemon=True).start()

flwr/{supernode/scheduler → supercore/cli}/__init__.py

@@ -12,11 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Flower ClientApp Scheduler."""
+"""Flower command line interface for shared infrastructure components."""
 
 
-from .simple_clientapp_scheduler_plugin import SimpleClientAppSchedulerPlugin
+from .flower_superexec import flower_superexec
 
 __all__ = [
-    "SimpleClientAppSchedulerPlugin",
+    "flower_superexec",
 ]

flwr/supercore/cli/flower_superexec.py

@@ -0,0 +1,141 @@
+# Copyright 2025 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""`flower-superexec` command."""
+
+
+import argparse
+from logging import INFO
+from typing import Optional
+
+from flwr.common import EventType, event
+from flwr.common.constant import ExecPluginType
+from flwr.common.exit import ExitCode, flwr_exit
+from flwr.common.logger import log
+from flwr.proto.clientappio_pb2_grpc import ClientAppIoStub
+from flwr.proto.serverappio_pb2_grpc import ServerAppIoStub
+from flwr.proto.simulationio_pb2_grpc import SimulationIoStub
+from flwr.supercore.grpc_health import add_args_health
+from flwr.supercore.superexec.plugin import (
+    ClientAppExecPlugin,
+    ExecPlugin,
+    ServerAppExecPlugin,
+    SimulationExecPlugin,
+)
+from flwr.supercore.superexec.run_superexec import run_superexec
+
+try:
+    from flwr.ee.constant import ExecEePluginType
+    from flwr.ee.exec_plugin import get_ee_plugin_and_stub_class
+except ImportError:
+
+    class ExecEePluginType:  # type: ignore[no-redef]
+        """SuperExec EE plugin types."""
+
+        @staticmethod
+        def all() -> list[str]:
+            """Return all SuperExec EE plugin types."""
+            return []
+
+    def get_ee_plugin_and_stub_class(  # pylint: disable=unused-argument
+        plugin_type: str,
+    ) -> Optional[tuple[type[ExecPlugin], type[object]]]:
+        """Get the EE plugin class and stub class based on the plugin type."""
+        return None
+
+
+def flower_superexec() -> None:
+    """Run `flower-superexec` command."""
+    args = _parse_args().parse_args()
+    if not args.insecure:
+        flwr_exit(
+            ExitCode.COMMON_TLS_NOT_SUPPORTED,
+            "SuperExec does not support TLS yet.",
+        )
+
+    # Log the first message after parsing arguments in case of `--help`
+    log(INFO, "Starting Flower SuperExec")
+
+    # Trigger telemetry event
+    event(EventType.RUN_SUPEREXEC_ENTER, {"plugin_type": args.plugin_type})
+
+    # Get the plugin class and stub class based on the plugin type
+    plugin_class, stub_class = _get_plugin_and_stub_class(args.plugin_type)
+    run_superexec(
+        plugin_class=plugin_class,
+        stub_class=stub_class,  # type: ignore
+        appio_api_address=args.appio_api_address,
+        flwr_dir=args.flwr_dir,
+        parent_pid=args.parent_pid,
+        health_server_address=args.health_server_address,
+    )
+
+
+def _parse_args() -> argparse.ArgumentParser:
+    """Parse `flower-superexec` command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Run Flower SuperExec.",
+    )
+    parser.add_argument(
+        "--appio-api-address", type=str, required=True, help="Address of the AppIO API"
+    )
+    parser.add_argument(
+        "--plugin-type",
+        type=str,
+        choices=ExecPluginType.all() + ExecEePluginType.all(),
+        required=True,
+        help="The type of plugin to use.",
+    )
+    parser.add_argument(
+        "--insecure",
+        action="store_true",
+        help="Connect to the AppIO API without TLS. "
+        "Data transmitted between the client and server is not encrypted. "
+        "Use this flag only if you understand the risks.",
+    )
+    parser.add_argument(
+        "--flwr-dir",
+        default=None,
+        help="""The path containing installed Flower Apps.
+        By default, this value is equal to:
+
+            - `$FLWR_HOME/` if `$FLWR_HOME` is defined
+            - `$XDG_DATA_HOME/.flwr/` if `$XDG_DATA_HOME` is defined
+            - `$HOME/.flwr/` in all other cases
+        """,
+    )
+    parser.add_argument(
+        "--parent-pid",
+        type=int,
+        default=None,
+        help="The PID of the parent process. When set, the process will terminate "
+        "when the parent process exits.",
+    )
+    add_args_health(parser)
+    return parser
+
+
+def _get_plugin_and_stub_class(
+    plugin_type: str,
+) -> tuple[type[ExecPlugin], type[object]]:
+    """Get the plugin class and stub class based on the plugin type."""
+    if plugin_type == ExecPluginType.CLIENT_APP:
+        return ClientAppExecPlugin, ClientAppIoStub
+    if plugin_type == ExecPluginType.SERVER_APP:
+        return ServerAppExecPlugin, ServerAppIoStub
+    if plugin_type == ExecPluginType.SIMULATION:
+        return SimulationExecPlugin, SimulationIoStub
+    if ret := get_ee_plugin_and_stub_class(plugin_type):
+        return ret  # type: ignore[no-any-return]
+    raise ValueError(f"Unknown plugin type: {plugin_type}")

flwr/supercore/{scheduler → corestate}/__init__.py

@@ -12,11 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Flower App Scheduler."""
+"""Flower CoreState."""
 
 
-from .plugin import SchedulerPlugin
+from .corestate import CoreState
 
 __all__ = [
-    "SchedulerPlugin",
+    "CoreState",
 ]