flwr-nightly 1.13.0.dev20241022__tar.gz → 1.13.0.dev20241023__tar.gz

{flwr_nightly-1.13.0.dev20241022 → flwr_nightly-1.13.0.dev20241023}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.13.0.dev20241022
+Version: 1.13.0.dev20241023
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0

{flwr_nightly-1.13.0.dev20241022 → flwr_nightly-1.13.0.dev20241023}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "flwr-nightly"
-version = "1.13.0.dev20241022"
+version = "1.13.0.dev20241023"
 description = "Flower: A Friendly Federated Learning Framework"
 license = "Apache-2.0"
 authors = ["The Flower Authors <hello@flower.ai>"]

{flwr_nightly-1.13.0.dev20241022 → flwr_nightly-1.13.0.dev20241023}/src/py/flwr/client/app.py

@@ -37,6 +37,8 @@ from flwr.common import GRPC_MAX_MESSAGE_LENGTH, Context, EventType, Message, ev
 from flwr.common.address import parse_address
 from flwr.common.constant import (
     CLIENTAPPIO_API_DEFAULT_ADDRESS,
+    ISOLATION_MODE_PROCESS,
+    ISOLATION_MODE_SUBPROCESS,
     MISSING_EXTRA_REST,
     RUN_ID_NUM_BYTES,
     TRANSPORT_TYPE_GRPC_ADAPTER,
@@ -62,9 +64,6 @@ from .message_handler.message_handler import handle_control_message
 from .numpy_client import NumPyClient
 from .run_info_store import DeprecatedRunInfoStore
 
-ISOLATION_MODE_SUBPROCESS = "subprocess"
-ISOLATION_MODE_PROCESS = "process"
-
 
 def _check_actionable_client(
     client: Optional[Client], client_fn: Optional[ClientFnExt]

{flwr_nightly-1.13.0.dev20241022 → flwr_nightly-1.13.0.dev20241023}/src/py/flwr/client/supernode/app.py

@@ -31,6 +31,8 @@ from flwr.common import EventType, event
 from flwr.common.config import parse_config_args
 from flwr.common.constant import (
     FLEET_API_GRPC_RERE_DEFAULT_ADDRESS,
+    ISOLATION_MODE_PROCESS,
+    ISOLATION_MODE_SUBPROCESS,
     TRANSPORT_TYPE_GRPC_ADAPTER,
     TRANSPORT_TYPE_GRPC_RERE,
     TRANSPORT_TYPE_REST,
@@ -38,11 +40,7 @@ from flwr.common.constant import (
 from flwr.common.exit_handlers import register_exit_handlers
 from flwr.common.logger import log, warn_deprecated_feature
 
-from ..app import (
-    ISOLATION_MODE_PROCESS,
-    ISOLATION_MODE_SUBPROCESS,
-    start_client_internal,
-)
+from ..app import start_client_internal
 from ..clientapp.utils import get_load_client_app_fn
 
 
@@ -200,10 +198,10 @@ def _parse_args_run_supernode() -> argparse.ArgumentParser:
             ISOLATION_MODE_SUBPROCESS,
             ISOLATION_MODE_PROCESS,
         ],
-        help="Isolation mode when running `ClientApp` (optional, possible values: "
-        "`subprocess`, `process`). By default, `ClientApp` runs in the same process "
+        help="Isolation mode when running a `ClientApp` (optional, possible values: "
+        "`subprocess`, `process`). By default, a `ClientApp` runs in the same process "
         "that executes the SuperNode. Use `subprocess` to configure SuperNode to run "
-        "`ClientApp` in a subprocess. Use `process` to indicate that a separate "
+        "a `ClientApp` in a subprocess. Use `process` to indicate that a separate "
         "independent process gets created outside of SuperNode.",
     )
     parser.add_argument(

{flwr_nightly-1.13.0.dev20241022 → flwr_nightly-1.13.0.dev20241023}/src/py/flwr/common/constant.py

@@ -83,6 +83,10 @@ GRPC_ADAPTER_METADATA_MESSAGE_QUALNAME_KEY = "grpc-message-qualname"
 # Message TTL
 MESSAGE_TTL_TOLERANCE = 1e-1
 
+# Isolation modes
+ISOLATION_MODE_SUBPROCESS = "subprocess"
+ISOLATION_MODE_PROCESS = "process"
+
 
 class MessageType:
     """Message type."""
@@ -128,3 +132,28 @@ class ErrorCode:
     def __new__(cls) -> ErrorCode:
         """Prevent instantiation."""
         raise TypeError(f"{cls.__name__} cannot be instantiated.")
+
+
+class Status:
+    """Run status."""
+
+    PENDING = "pending"
+    STARTING = "starting"
+    RUNNING = "running"
+    FINISHED = "finished"
+
+    def __new__(cls) -> Status:
+        """Prevent instantiation."""
+        raise TypeError(f"{cls.__name__} cannot be instantiated.")
+
+
+class SubStatus:
+    """Run sub-status."""
+
+    COMPLETED = "completed"
+    FAILED = "failed"
+    STOPPED = "stopped"
+
+    def __new__(cls) -> SubStatus:
+        """Prevent instantiation."""
+        raise TypeError(f"{cls.__name__} cannot be instantiated.")

{flwr_nightly-1.13.0.dev20241022 → flwr_nightly-1.13.0.dev20241023}/src/py/flwr/common/typing.py

@@ -218,6 +218,15 @@ class Run:
     override_config: UserConfig
 
 
+@dataclass
+class RunStatus:
+    """Run status information."""
+
+    status: str
+    sub_status: str
+    details: str
+
+
 @dataclass
 class Fab:
     """Fab file representation."""

{flwr_nightly-1.13.0.dev20241022 → flwr_nightly-1.13.0.dev20241023}/src/py/flwr/server/app.py

@@ -17,12 +17,14 @@
 import argparse
 import csv
 import importlib.util
+import subprocess
 import sys
 import threading
 from collections.abc import Sequence
-from logging import INFO, WARN
+from logging import DEBUG, INFO, WARN
 from os.path import isfile
 from pathlib import Path
+from time import sleep
 from typing import Optional
 
 import grpc
@@ -42,10 +44,13 @@ from flwr.common.constant import (
     FLEET_API_GRPC_BIDI_DEFAULT_ADDRESS,
     FLEET_API_GRPC_RERE_DEFAULT_ADDRESS,
     FLEET_API_REST_DEFAULT_ADDRESS,
+    ISOLATION_MODE_PROCESS,
+    ISOLATION_MODE_SUBPROCESS,
     MISSING_EXTRA_REST,
     TRANSPORT_TYPE_GRPC_ADAPTER,
     TRANSPORT_TYPE_GRPC_RERE,
     TRANSPORT_TYPE_REST,
+    Status,
 )
 from flwr.common.exit_handlers import register_exit_handlers
 from flwr.common.logger import log
@@ -53,6 +58,7 @@ from flwr.common.secure_aggregation.crypto.symmetric_encryption import (
     private_key_to_bytes,
     public_key_to_bytes,
 )
+from flwr.common.typing import RunStatus
 from flwr.proto.fleet_pb2_grpc import (  # pylint: disable=E0611
     add_FleetServicer_to_server,
 )
@@ -333,6 +339,15 @@ def run_superlink() -> None:
     )
     grpc_servers.append(exec_server)
 
+    if args.isolation == ISOLATION_MODE_SUBPROCESS:
+        # Scheduler thread
+        scheduler_th = threading.Thread(
+            target=_flwr_serverapp_scheduler,
+            args=(state_factory, args.driver_api_address),
+        )
+        scheduler_th.start()
+        bckg_threads.append(scheduler_th)
+
     # Graceful shutdown
     register_exit_handlers(
         event_type=EventType.RUN_SUPERLINK_LEAVE,
@@ -349,6 +364,47 @@ def run_superlink() -> None:
         driver_server.wait_for_termination(timeout=1)
 
 
+def _flwr_serverapp_scheduler(
+    state_factory: LinkStateFactory, driver_api_address: str
+) -> None:
+    log(DEBUG, "Started flwr-serverapp scheduler thread.")
+
+    state = state_factory.state()
+
+    # Periodically check for a pending run in the LinkState
+    while True:
+        sleep(3)
+        pending_run_id = state.get_pending_run_id()
+
+        if pending_run_id:
+
+            # Set run as starting
+            state.update_run_status(
+                run_id=pending_run_id, new_status=RunStatus(Status.STARTING, "", "")
+            )
+            log(
+                INFO,
+                "Launching `flwr-serverapp` subprocess with run-id %d. "
+                "Connects to SuperLink on %s",
+                pending_run_id,
+                driver_api_address,
+            )
+            # Start ServerApp subprocess
+            command = [
+                "flwr-serverapp",
+                "--superlink",
+                driver_api_address,
+                "--run-id",
+                str(pending_run_id),
+            ]
+            subprocess.run(
+                command,
+                stdout=None,
+                stderr=None,
+                check=True,
+            )
+
+
 def _format_address(address: str) -> tuple[str, str, int]:
     parsed_address = parse_address(address)
     if not parsed_address:
@@ -634,6 +690,19 @@ def _add_args_common(parser: argparse.ArgumentParser) -> None:
         "to create a secure connection.",
         type=str,
     )
+    parser.add_argument(
+        "--isolation",
+        default=ISOLATION_MODE_SUBPROCESS,
+        required=False,
+        choices=[
+            ISOLATION_MODE_SUBPROCESS,
+            ISOLATION_MODE_PROCESS,
+        ],
+        help="Isolation mode when running a `ServerApp` (`subprocess` by default, "
+        "possible values: `subprocess`, `process`). Use `subprocess` to configure "
+        "SuperLink to run a `ServerApp` in a subprocess. Use `process` to indicate "
+        "that a separate independent process gets created outside of SuperLink.",
+    )
     parser.add_argument(
         "--database",
         help="A string representing the path to the database "

flwr_nightly-1.13.0.dev20241023/src/py/flwr/server/serverapp/app.py

@@ -0,0 +1,78 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower ServerApp process."""
+
+import argparse
+from logging import DEBUG, INFO
+from typing import Optional
+
+from flwr.common.logger import log
+from flwr.server.driver.grpc_driver import GrpcDriver
+
+
+def flwr_serverapp() -> None:
+    """Run process-isolated Flower ServerApp."""
+    log(INFO, "Starting Flower ServerApp")
+
+    parser = argparse.ArgumentParser(
+        description="Run a Flower ServerApp",
+    )
+    parser.add_argument(
+        "--superlink",
+        type=str,
+        help="Address of SuperLink's DriverAPI",
+    )
+    parser.add_argument(
+        "--run-id",
+        type=int,
+        required=False,
+        help="Id of the Run this process should start. If not supplied, this "
+        "function will request a pending run to the LinkState.",
+    )
+    args = parser.parse_args()
+
+    log(
+        DEBUG,
+        "Staring isolated `ServerApp` connected to SuperLink DriverAPI at %s "
+        "for run-id %s",
+        args.superlink,
+        args.run_id,
+    )
+    run_serverapp(superlink=args.superlink, run_id=args.run_id)
+
+
+def run_serverapp(  # pylint: disable=R0914
+    superlink: str,
+    run_id: Optional[int] = None,
+) -> None:
+    """Run Flower ServerApp process.
+
+    Parameters
+    ----------
+    superlink : str
+        Address of SuperLink
+    run_id : Optional[int] (default: None)
+        Unique identifier of a Run registered at the LinkState. If not supplied,
+        this function will request a pending run to the LinkState.
+    """
+    _ = GrpcDriver(
+        run_id=run_id if run_id else 0,
+        driver_service_address=superlink,
+        root_certificates=None,
+    )
+
+    # Then, GetServerInputs
+
+    # Then, run ServerApp

{flwr_nightly-1.13.0.dev20241022 → flwr_nightly-1.13.0.dev20241023}/src/py/flwr/server/superlink/linkstate/in_memory_linkstate.py

@@ -17,6 +17,7 @@
 
 import threading
 import time
+from dataclasses import dataclass
 from logging import ERROR, WARNING
 from typing import Optional
 from uuid import UUID, uuid4
@@ -26,13 +27,31 @@ from flwr.common.constant import (
     MESSAGE_TTL_TOLERANCE,
     NODE_ID_NUM_BYTES,
     RUN_ID_NUM_BYTES,
+    Status,
 )
-from flwr.common.typing import Run, UserConfig
+from flwr.common.typing import Run, RunStatus, UserConfig
 from flwr.proto.task_pb2 import TaskIns, TaskRes  # pylint: disable=E0611
 from flwr.server.superlink.linkstate.linkstate import LinkState
 from flwr.server.utils import validate_task_ins_or_res
 
-from .utils import generate_rand_int_from_bytes, make_node_unavailable_taskres
+from .utils import (
+    generate_rand_int_from_bytes,
+    has_valid_sub_status,
+    is_valid_transition,
+    make_node_unavailable_taskres,
+)
+
+
+@dataclass
+class RunRecord:
+    """The record of a specific run, including its status and timestamps."""
+
+    run: Run
+    status: RunStatus
+    pending_at: str = ""
+    starting_at: str = ""
+    running_at: str = ""
+    finished_at: str = ""
 
 
 class InMemoryLinkState(LinkState):  # pylint: disable=R0902,R0904
@@ -44,8 +63,8 @@ class InMemoryLinkState(LinkState):  # pylint: disable=R0902,R0904
         self.node_ids: dict[int, tuple[float, float]] = {}
         self.public_key_to_node_id: dict[bytes, int] = {}
 
-        # Map run_id to (fab_id, fab_version)
-        self.run_ids: dict[int, Run] = {}
+        # Map run_id to RunRecord
+        self.run_ids: dict[int, RunRecord] = {}
         self.task_ins_store: dict[UUID, TaskIns] = {}
         self.task_res_store: dict[UUID, TaskRes] = {}
 
@@ -351,13 +370,22 @@ class InMemoryLinkState(LinkState):  # pylint: disable=R0902,R0904
             run_id = generate_rand_int_from_bytes(RUN_ID_NUM_BYTES)
 
             if run_id not in self.run_ids:
-                self.run_ids[run_id] = Run(
-                    run_id=run_id,
-                    fab_id=fab_id if fab_id else "",
-                    fab_version=fab_version if fab_version else "",
-                    fab_hash=fab_hash if fab_hash else "",
-                    override_config=override_config,
+                run_record = RunRecord(
+                    run=Run(
+                        run_id=run_id,
+                        fab_id=fab_id if fab_id else "",
+                        fab_version=fab_version if fab_version else "",
+                        fab_hash=fab_hash if fab_hash else "",
+                        override_config=override_config,
+                    ),
+                    status=RunStatus(
+                        status=Status.PENDING,
+                        sub_status="",
+                        details="",
+                    ),
+                    pending_at=now().isoformat(),
                 )
+                self.run_ids[run_id] = run_record
                 return run_id
         log(ERROR, "Unexpected run creation failure.")
         return 0
@@ -401,7 +429,69 @@ class InMemoryLinkState(LinkState):  # pylint: disable=R0902,R0904
             if run_id not in self.run_ids:
                 log(ERROR, "`run_id` is invalid")
                 return None
-            return self.run_ids[run_id]
+            return self.run_ids[run_id].run
+
+    def get_run_status(self, run_ids: set[int]) -> dict[int, RunStatus]:
+        """Retrieve the statuses for the specified runs."""
+        with self.lock:
+            return {
+                run_id: self.run_ids[run_id].status
+                for run_id in set(run_ids)
+                if run_id in self.run_ids
+            }
+
+    def update_run_status(self, run_id: int, new_status: RunStatus) -> bool:
+        """Update the status of the run with the specified `run_id`."""
+        with self.lock:
+            # Check if the run_id exists
+            if run_id not in self.run_ids:
+                log(ERROR, "`run_id` is invalid")
+                return False
+
+            # Check if the status transition is valid
+            current_status = self.run_ids[run_id].status
+            if not is_valid_transition(current_status, new_status):
+                log(
+                    ERROR,
+                    'Invalid status transition: from "%s" to "%s"',
+                    current_status.status,
+                    new_status.status,
+                )
+                return False
+
+            # Check if the sub-status is valid
+            if not has_valid_sub_status(current_status):
+                log(
+                    ERROR,
+                    'Invalid sub-status "%s" for status "%s"',
+                    current_status.sub_status,
+                    current_status.status,
+                )
+                return False
+
+            # Update the status
+            run_record = self.run_ids[run_id]
+            if new_status.status == Status.STARTING:
+                run_record.starting_at = now().isoformat()
+            elif new_status.status == Status.RUNNING:
+                run_record.running_at = now().isoformat()
+            elif new_status.status == Status.FINISHED:
+                run_record.finished_at = now().isoformat()
+            run_record.status = new_status
+            return True
+
+    def get_pending_run_id(self) -> Optional[int]:
+        """Get the `run_id` of a run with `Status.PENDING` status, if any."""
+        pending_run_id = None
+
+        # Loop through all registered runs
+        for run_id, run_rec in self.run_ids.items():
+            # Break once a pending run is found
+            if run_rec.status.status == Status.PENDING:
+                pending_run_id = run_id
+                break
+
+        return pending_run_id
 
     def acknowledge_ping(self, node_id: int, ping_interval: float) -> bool:
         """Acknowledge a ping received from a node, serving as a heartbeat."""

{flwr_nightly-1.13.0.dev20241022 → flwr_nightly-1.13.0.dev20241023}/src/py/flwr/server/superlink/linkstate/linkstate.py

@@ -19,7 +19,7 @@ import abc
 from typing import Optional
 from uuid import UUID
 
-from flwr.common.typing import Run, UserConfig
+from flwr.common.typing import Run, RunStatus, UserConfig
 from flwr.proto.task_pb2 import TaskIns, TaskRes  # pylint: disable=E0611
 
 
@@ -178,6 +178,54 @@ class LinkState(abc.ABC):  # pylint: disable=R0904
             - `fab_version`: The version of the FAB used in the specified run.
         """
 
+    @abc.abstractmethod
+    def get_run_status(self, run_ids: set[int]) -> dict[int, RunStatus]:
+        """Retrieve the statuses for the specified runs.
+
+        Parameters
+        ----------
+        run_ids : set[int]
+            A set of run identifiers for which to retrieve statuses.
+
+        Returns
+        -------
+        dict[int, RunStatus]
+            A dictionary mapping each valid run ID to its corresponding status.
+
+        Notes
+        -----
+        Only valid run IDs that exist in the State will be included in the returned
+        dictionary. If a run ID is not found, it will be omitted from the result.
+        """
+
+    @abc.abstractmethod
+    def update_run_status(self, run_id: int, new_status: RunStatus) -> bool:
+        """Update the status of the run with the specified `run_id`.
+
+        Parameters
+        ----------
+        run_id : int
+            The identifier of the run.
+        new_status : RunStatus
+            The new status to be assigned to the run.
+
+        Returns
+        -------
+        bool
+            True if the status update is successful; False otherwise.
+        """
+
+    @abc.abstractmethod
+    def get_pending_run_id(self) -> Optional[int]:
+        """Get the `run_id` of a run with `Status.PENDING` status.
+
+        Returns
+        -------
+        Optional[int]
+            The `run_id` of a `Run` that is pending to be started; None if
+            there is no Run pending.
+        """
+
     @abc.abstractmethod
     def store_server_private_public_key(
         self, private_key: bytes, public_key: bytes