flwr-nightly 1.13.0.dev20241105__py3-none-any.whl → 1.13.0.dev20241107__py3-none-any.whl

flwr/server/superlink/linkstate/utils.py

@@ -15,18 +15,15 @@
 """Utility functions for State."""
 
 
-import time
-from logging import ERROR
 from os import urandom
-from uuid import uuid4
 
-from flwr.common import Context, log, serde
-from flwr.common.constant import ErrorCode, Status, SubStatus
+from flwr.common import ConfigsRecord, Context, serde
+from flwr.common.constant import Status, SubStatus
 from flwr.common.typing import RunStatus
-from flwr.proto.error_pb2 import Error  # pylint: disable=E0611
 from flwr.proto.message_pb2 import Context as ProtoContext  # pylint: disable=E0611
-from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
-from flwr.proto.task_pb2 import Task, TaskIns, TaskRes  # pylint: disable=E0611
+
+# pylint: disable=E0611
+from flwr.proto.recordset_pb2 import ConfigsRecord as ProtoConfigsRecord
 
 NODE_UNAVAILABLE_ERROR_REASON = (
     "Error: Node Unavailable - The destination node is currently unavailable. "
@@ -146,28 +143,15 @@ def context_from_bytes(context_bytes: bytes) -> Context:
     return serde.context_from_proto(ProtoContext.FromString(context_bytes))
 
 
-def make_node_unavailable_taskres(ref_taskins: TaskIns) -> TaskRes:
-    """Generate a TaskRes with a node unavailable error from a TaskIns."""
-    current_time = time.time()
-    ttl = ref_taskins.task.ttl - (current_time - ref_taskins.task.created_at)
-    if ttl < 0:
-        log(ERROR, "Creating TaskRes for TaskIns that exceeds its TTL.")
-        ttl = 0
-    return TaskRes(
-        task_id=str(uuid4()),
-        group_id=ref_taskins.group_id,
-        run_id=ref_taskins.run_id,
-        task=Task(
-            producer=Node(node_id=ref_taskins.task.consumer.node_id, anonymous=False),
-            consumer=Node(node_id=ref_taskins.task.producer.node_id, anonymous=False),
-            created_at=current_time,
-            ttl=ttl,
-            ancestry=[ref_taskins.task_id],
-            task_type=ref_taskins.task.task_type,
-            error=Error(
-                code=ErrorCode.NODE_UNAVAILABLE, reason=NODE_UNAVAILABLE_ERROR_REASON
-            ),
-        ),
+def configsrecord_to_bytes(configs_record: ConfigsRecord) -> bytes:
+    """Serialize a `ConfigsRecord` to bytes."""
+    return serde.configs_record_to_proto(configs_record).SerializeToString()
+
+
+def configsrecord_from_bytes(configsrecord_bytes: bytes) -> ConfigsRecord:
+    """Deserialize `ConfigsRecord` from bytes."""
+    return serde.configs_record_from_proto(
+        ProtoConfigsRecord.FromString(configsrecord_bytes)
     )
 
 

flwr/server/superlink/simulation/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower SimulationIo service."""

flwr/server/superlink/simulation/simulationio_grpc.py

@@ -0,0 +1,65 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""SimulationIo gRPC API."""
+
+
+from logging import INFO
+from typing import Optional
+
+import grpc
+
+from flwr.common import GRPC_MAX_MESSAGE_LENGTH
+from flwr.common.logger import log
+from flwr.proto.simulationio_pb2_grpc import (  # pylint: disable=E0611
+    add_SimulationIoServicer_to_server,
+)
+from flwr.server.superlink.ffs.ffs_factory import FfsFactory
+from flwr.server.superlink.linkstate import LinkStateFactory
+
+from ..fleet.grpc_bidi.grpc_server import generic_create_grpc_server
+from .simulationio_servicer import SimulationIoServicer
+
+
+def run_simulationio_api_grpc(
+    address: str,
+    state_factory: LinkStateFactory,
+    ffs_factory: FfsFactory,
+    certificates: Optional[tuple[bytes, bytes, bytes]],
+) -> grpc.Server:
+    """Run SimulationIo API (gRPC, request-response)."""
+    # Create SimulationIo API gRPC server
+    simulationio_servicer: grpc.Server = SimulationIoServicer(
+        state_factory=state_factory,
+        ffs_factory=ffs_factory,
+    )
+    simulationio_add_servicer_to_server_fn = add_SimulationIoServicer_to_server
+    simulationio_grpc_server = generic_create_grpc_server(
+        servicer_and_add_fn=(
+            simulationio_servicer,
+            simulationio_add_servicer_to_server_fn,
+        ),
+        server_address=address,
+        max_message_length=GRPC_MAX_MESSAGE_LENGTH,
+        certificates=certificates,
+    )
+
+    log(
+        INFO,
+        "Flower Simulation Engine: Starting SimulationIo API on %s",
+        address,
+    )
+    simulationio_grpc_server.start()
+
+    return simulationio_grpc_server

flwr/server/superlink/simulation/simulationio_servicer.py

@@ -0,0 +1,132 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""SimulationIo API servicer."""
+
+import threading
+from logging import DEBUG, INFO
+
+import grpc
+from grpc import ServicerContext
+
+from flwr.common.constant import Status
+from flwr.common.logger import log
+from flwr.common.serde import (
+    context_from_proto,
+    context_to_proto,
+    fab_to_proto,
+    run_status_from_proto,
+    run_to_proto,
+)
+from flwr.common.typing import Fab, RunStatus
+from flwr.proto import simulationio_pb2_grpc
+from flwr.proto.log_pb2 import (  # pylint: disable=E0611
+    PushLogsRequest,
+    PushLogsResponse,
+)
+from flwr.proto.run_pb2 import (  # pylint: disable=E0611
+    UpdateRunStatusRequest,
+    UpdateRunStatusResponse,
+)
+from flwr.proto.simulationio_pb2 import (  # pylint: disable=E0611
+    PullSimulationInputsRequest,
+    PullSimulationInputsResponse,
+    PushSimulationOutputsRequest,
+    PushSimulationOutputsResponse,
+)
+from flwr.server.superlink.ffs.ffs_factory import FfsFactory
+from flwr.server.superlink.linkstate import LinkStateFactory
+
+
+class SimulationIoServicer(simulationio_pb2_grpc.SimulationIoServicer):
+    """SimulationIo API servicer."""
+
+    def __init__(
+        self, state_factory: LinkStateFactory, ffs_factory: FfsFactory
+    ) -> None:
+        self.state_factory = state_factory
+        self.ffs_factory = ffs_factory
+        self.lock = threading.RLock()
+
+    def PullSimulationInputs(
+        self, request: PullSimulationInputsRequest, context: ServicerContext
+    ) -> PullSimulationInputsResponse:
+        """Pull SimultionIo process inputs."""
+        log(DEBUG, "SimultionIoServicer.SimultionIoInputs")
+        # Init access to LinkState and Ffs
+        state = self.state_factory.state()
+        ffs = self.ffs_factory.ffs()
+
+        # Lock access to LinkState, preventing obtaining the same pending run_id
+        with self.lock:
+            # Attempt getting the run_id of a pending run
+            run_id = state.get_pending_run_id()
+            # If there's no pending run, return an empty response
+            if run_id is None:
+                return PullSimulationInputsResponse()
+
+            # Retrieve Context, Run and Fab for the run_id
+            serverapp_ctxt = state.get_serverapp_context(run_id)
+            run = state.get_run(run_id)
+            fab = None
+            if run and run.fab_hash:
+                if result := ffs.get(run.fab_hash):
+                    fab = Fab(run.fab_hash, result[0])
+            if run and fab and serverapp_ctxt:
+                # Update run status to STARTING
+                if state.update_run_status(run_id, RunStatus(Status.STARTING, "", "")):
+                    log(INFO, "Starting run %d", run_id)
+                    return PullSimulationInputsResponse(
+                        context=context_to_proto(serverapp_ctxt),
+                        run=run_to_proto(run),
+                        fab=fab_to_proto(fab),
+                    )
+
+        # Raise an exception if the Run or Fab is not found,
+        # or if the status cannot be updated to STARTING
+        raise RuntimeError(f"Failed to start run {run_id}")
+
+    def PushSimulationOutputs(
+        self, request: PushSimulationOutputsRequest, context: ServicerContext
+    ) -> PushSimulationOutputsResponse:
+        """Push Simulation process outputs."""
+        log(DEBUG, "SimultionIoServicer.PushSimulationOutputs")
+        state = self.state_factory.state()
+        state.set_serverapp_context(request.run_id, context_from_proto(request.context))
+        return PushSimulationOutputsResponse()
+
+    def UpdateRunStatus(
+        self, request: UpdateRunStatusRequest, context: grpc.ServicerContext
+    ) -> UpdateRunStatusResponse:
+        """Update the status of a run."""
+        log(DEBUG, "SimultionIoServicer.UpdateRunStatus")
+        state = self.state_factory.state()
+
+        # Update the run status
+        state.update_run_status(
+            run_id=request.run_id, new_status=run_status_from_proto(request.run_status)
+        )
+        return UpdateRunStatusResponse()
+
+    def PushLogs(
+        self, request: PushLogsRequest, context: grpc.ServicerContext
+    ) -> PushLogsResponse:
+        """Push logs."""
+        log(DEBUG, "ServerAppIoServicer.PushLogs")
+        state = self.state_factory.state()
+
+        # Add logs to LinkState
+        merged_logs = "".join(request.logs)
+        state.add_serverapp_log(request.run_id, merged_logs)
+        return PushLogsResponse()

flwr/simulation/__init__.py

@@ -18,6 +18,7 @@
 import importlib
 
 from flwr.simulation.run_simulation import run_simulation
+from flwr.simulation.simulationio_connection import SimulationIoConnection
 
 is_ray_installed = importlib.util.find_spec("ray") is not None
 
@@ -37,6 +38,7 @@ To install the necessary dependencies, install `flwr` with the `simulation` extr
 
 
 __all__ = [
+    "SimulationIoConnection",
     "run_simulation",
     "start_simulation",
 ]

flwr/simulation/run_simulation.py

@@ -234,6 +234,7 @@ def run_serverapp_th(
     f_stop: threading.Event,
     has_exception: threading.Event,
     enable_tf_gpu_growth: bool,
+    run_id: int,
 ) -> threading.Thread:
     """Run SeverApp in a thread."""
 
@@ -258,6 +259,7 @@ def run_serverapp_th(
 
             # Initialize Context
             context = Context(
+                run_id=run_id,
                 node_id=0,
                 node_config={},
                 state=RecordSet(),
@@ -345,7 +347,7 @@ def _main_loop(
 
         # Initialize Driver
         driver = InMemoryDriver(state_factory=state_factory)
-        driver.init_run(run_id=run.run_id)
+        driver.set_run(run_id=run.run_id)
 
         # Get and run ServerApp thread
         serverapp_th = run_serverapp_th(
@@ -357,6 +359,7 @@ def _main_loop(
             f_stop=f_stop,
             has_exception=server_app_thread_has_exception,
             enable_tf_gpu_growth=enable_tf_gpu_growth,
+            run_id=run.run_id,
         )
 
         # Buffer time so the `ServerApp` in separate thread is ready

flwr/simulation/simulationio_connection.py

@@ -0,0 +1,86 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower SimulationIo connection."""
+
+
+from logging import DEBUG, WARNING
+from typing import Optional, cast
+
+import grpc
+
+from flwr.common.constant import SIMULATIONIO_API_DEFAULT_ADDRESS
+from flwr.common.grpc import create_channel
+from flwr.common.logger import log
+from flwr.proto.simulationio_pb2_grpc import SimulationIoStub  # pylint: disable=E0611
+
+
+class SimulationIoConnection:
+    """`SimulationIoConnection` provides an interface to the SimulationIo API.
+
+    Parameters
+    ----------
+    simulationio_service_address : str (default: "[::]:9094")
+        The address (URL, IPv6, IPv4) of the SuperLink SimulationIo API service.
+    root_certificates : Optional[bytes] (default: None)
+        The PEM-encoded root certificates as a byte string.
+        If provided, a secure connection using the certificates will be
+        established to an SSL-enabled Flower server.
+    """
+
+    def __init__(  # pylint: disable=too-many-arguments
+        self,
+        simulationio_service_address: str = SIMULATIONIO_API_DEFAULT_ADDRESS,
+        root_certificates: Optional[bytes] = None,
+    ) -> None:
+        self._addr = simulationio_service_address
+        self._cert = root_certificates
+        self._grpc_stub: Optional[SimulationIoStub] = None
+        self._channel: Optional[grpc.Channel] = None
+
+    @property
+    def _is_connected(self) -> bool:
+        """Check if connected to the SimulationIo API server."""
+        return self._channel is not None
+
+    @property
+    def _stub(self) -> SimulationIoStub:
+        """SimulationIo stub."""
+        if not self._is_connected:
+            self._connect()
+        return cast(SimulationIoStub, self._grpc_stub)
+
+    def _connect(self) -> None:
+        """Connect to the SimulationIo API."""
+        if self._is_connected:
+            log(WARNING, "Already connected")
+            return
+        self._channel = create_channel(
+            server_address=self._addr,
+            insecure=(self._cert is None),
+            root_certificates=self._cert,
+        )
+        self._grpc_stub = SimulationIoStub(self._channel)
+        log(DEBUG, "[SimulationIO] Connected to %s", self._addr)
+
+    def _disconnect(self) -> None:
+        """Disconnect from the SimulationIo API."""
+        if not self._is_connected:
+            log(DEBUG, "Already disconnected")
+            return
+        channel: grpc.Channel = self._channel
+        self._channel = None
+        self._grpc_stub = None
+        channel.close()
+        log(DEBUG, "[SimulationIO] Disconnected")

flwr/superexec/deployment.py

@@ -21,7 +21,7 @@ from typing import Optional
 
 from typing_extensions import override
 
-from flwr.common import Context, RecordSet
+from flwr.common import ConfigsRecord, Context, RecordSet
 from flwr.common.constant import SERVERAPPIO_API_DEFAULT_ADDRESS, Status, SubStatus
 from flwr.common.logger import log
 from flwr.common.typing import Fab, RunStatus, UserConfig
@@ -133,13 +133,17 @@ class DeploymentEngine(Executor):
                 f"FAB ({fab.hash_str}) hash from request doesn't match contents"
             )
 
-        run_id = self.linkstate.create_run(None, None, fab_hash, override_config)
+        run_id = self.linkstate.create_run(
+            None, None, fab_hash, override_config, ConfigsRecord()
+        )
         return run_id
 
     def _create_context(self, run_id: int) -> None:
         """Register a Context for a Run."""
         # Create an empty context for the Run
-        context = Context(node_id=0, node_config={}, state=RecordSet(), run_config={})
+        context = Context(
+            run_id=run_id, node_id=0, node_config={}, state=RecordSet(), run_config={}
+        )
 
         # Register the context at the LinkState
         self.linkstate.set_serverapp_context(run_id=run_id, context=context)
@@ -149,7 +153,7 @@ class DeploymentEngine(Executor):
         self,
         fab_file: bytes,
         override_config: UserConfig,
-        federation_config: UserConfig,
+        federation_options: ConfigsRecord,
     ) -> Optional[int]:
         """Start run using the Flower Deployment Engine."""
         run_id = None

flwr/superexec/exec_servicer.py

@@ -24,7 +24,7 @@ import grpc
 
 from flwr.common.constant import LOG_STREAM_INTERVAL, Status
 from flwr.common.logger import log
-from flwr.common.serde import user_config_from_proto
+from flwr.common.serde import configs_record_from_proto, user_config_from_proto
 from flwr.proto import exec_pb2_grpc  # pylint: disable=E0611
 from flwr.proto.exec_pb2 import (  # pylint: disable=E0611
     StartRunRequest,
@@ -61,7 +61,7 @@ class ExecServicer(exec_pb2_grpc.ExecServicer):
         run_id = self.executor.start_run(
             request.fab.content,
             user_config_from_proto(request.override_config),
-            user_config_from_proto(request.federation_config),
+            configs_record_from_proto(request.federation_options),
         )
 
         if run_id is None:

flwr/superexec/executor.py

@@ -19,6 +19,7 @@ from dataclasses import dataclass, field
 from subprocess import Popen
 from typing import Optional
 
+from flwr.common import ConfigsRecord
 from flwr.common.typing import UserConfig
 from flwr.server.superlink.ffs.ffs_factory import FfsFactory
 from flwr.server.superlink.linkstate import LinkStateFactory
@@ -71,7 +72,7 @@ class Executor(ABC):
         self,
         fab_file: bytes,
         override_config: UserConfig,
-        federation_config: UserConfig,
+        federation_options: ConfigsRecord,
     ) -> Optional[int]:
         """Start a run using the given Flower FAB ID and version.
 
@@ -84,8 +85,8 @@ class Executor(ABC):
             The Flower App Bundle file bytes.
         override_config: UserConfig
             The config overrides dict sent by the user (using `flwr run`).
-        federation_config: UserConfig
-            The federation options dict sent by the user (using `flwr run`).
+        federation_options: ConfigsRecord
+            The federation options sent by the user (using `flwr run`).
 
         Returns
         -------

flwr/superexec/simulation.py

@@ -25,6 +25,7 @@ from typing_extensions import override
 
 from flwr.cli.config_utils import load_and_validate
 from flwr.cli.install import install_from_fab
+from flwr.common import ConfigsRecord
 from flwr.common.config import unflatten_dict
 from flwr.common.constant import RUN_ID_NUM_BYTES
 from flwr.common.logger import log
@@ -124,7 +125,7 @@ class SimulationEngine(Executor):
         self,
         fab_file: bytes,
         override_config: UserConfig,
-        federation_config: UserConfig,
+        federation_options: ConfigsRecord,
     ) -> Optional[int]:
         """Start run using the Flower Simulation Engine."""
         if self.num_supernodes is None:
@@ -163,14 +164,13 @@ class SimulationEngine(Executor):
                     "Config extracted from FAB's pyproject.toml is not valid"
                 )
 
-            # Flatten federated config
-            federation_config_flat = unflatten_dict(federation_config)
+            # Unflatten underlaying dict
+            fed_opt = unflatten_dict({**federation_options})
 
-            num_supernodes = federation_config_flat.get(
-                "num-supernodes", self.num_supernodes
-            )
-            backend_cfg = federation_config_flat.get("backend", {})
-            verbose: Optional[bool] = federation_config_flat.get("verbose")
+            # Read data
+            num_supernodes = fed_opt.get("num-supernodes", self.num_supernodes)
+            backend_cfg = fed_opt.get("backend", {})
+            verbose: Optional[bool] = fed_opt.get("verbose")
 
             # In Simulation there is no SuperLink, still we create a run_id
             run_id = generate_rand_int_from_bytes(RUN_ID_NUM_BYTES)

{flwr_nightly-1.13.0.dev20241105.dist-info → flwr_nightly-1.13.0.dev20241107.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.13.0.dev20241105
+Version: 1.13.0.dev20241107
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0