flwr-nightly 1.13.0.dev20241021__py3-none-any.whl → 1.13.0.dev20241111__py3-none-any.whl

flwr/superexec/deployment.py

@@ -15,23 +15,21 @@
 """Deployment engine executor."""
 
 import hashlib
-import subprocess
 from logging import ERROR, INFO
 from pathlib import Path
 from typing import Optional
 
 from typing_extensions import override
 
-from flwr.cli.install import install_from_fab
-from flwr.common.constant import DRIVER_API_DEFAULT_ADDRESS
-from flwr.common.grpc import create_channel
+from flwr.common import ConfigsRecord, Context, RecordSet
+from flwr.common.constant import SERVERAPPIO_API_DEFAULT_ADDRESS, Status, SubStatus
 from flwr.common.logger import log
-from flwr.common.serde import fab_to_proto, user_config_to_proto
-from flwr.common.typing import Fab, UserConfig
-from flwr.proto.driver_pb2_grpc import DriverStub
-from flwr.proto.run_pb2 import CreateRunRequest  # pylint: disable=E0611
+from flwr.common.typing import Fab, RunStatus, UserConfig
+from flwr.server.superlink.ffs import Ffs
+from flwr.server.superlink.ffs.ffs_factory import FfsFactory
+from flwr.server.superlink.linkstate import LinkState, LinkStateFactory
 
-from .executor import Executor, RunTracker
+from .executor import Executor
 
 
 class DeploymentEngine(Executor):
@@ -50,7 +48,7 @@ class DeploymentEngine(Executor):
 
     def __init__(
         self,
-        superlink: str = DRIVER_API_DEFAULT_ADDRESS,
+        superlink: str = SERVERAPPIO_API_DEFAULT_ADDRESS,
         root_certificates: Optional[str] = None,
         flwr_dir: Optional[str] = None,
     ) -> None:
@@ -62,7 +60,30 @@ class DeploymentEngine(Executor):
             self.root_certificates = root_certificates
             self.root_certificates_bytes = Path(root_certificates).read_bytes()
         self.flwr_dir = flwr_dir
-        self.stub: Optional[DriverStub] = None
+        self.linkstate_factory: Optional[LinkStateFactory] = None
+        self.ffs_factory: Optional[FfsFactory] = None
+
+    @override
+    def initialize(
+        self, linkstate_factory: LinkStateFactory, ffs_factory: FfsFactory
+    ) -> None:
+        """Initialize the executor with the necessary factories."""
+        self.linkstate_factory = linkstate_factory
+        self.ffs_factory = ffs_factory
+
+    @property
+    def linkstate(self) -> LinkState:
+        """Return the LinkState."""
+        if self.linkstate_factory is None:
+            raise RuntimeError("Executor is not initialized.")
+        return self.linkstate_factory.state()
+
+    @property
+    def ffs(self) -> Ffs:
+        """Return the Flower File Storage (FFS)."""
+        if self.ffs_factory is None:
+            raise RuntimeError("Executor is not initialized.")
+        return self.ffs_factory.ffs()
 
     @override
     def set_config(
@@ -77,7 +98,7 @@ class DeploymentEngine(Executor):
             A dictionary for configuration values.
             Supported configuration key/value pairs:
             - "superlink": str
-                The address of the SuperLink Driver API.
+                The address of the SuperLink ServerAppIo API.
             - "root-certificates": str
                 The path to the root certificates.
             - "flwr-dir": str
@@ -101,85 +122,59 @@ class DeploymentEngine(Executor):
                 raise ValueError("The `flwr-dir` value should be of type `str`.")
             self.flwr_dir = str(flwr_dir)
 
-    def _connect(self) -> None:
-        if self.stub is not None:
-            return
-        channel = create_channel(
-            server_address=self.superlink,
-            insecure=(self.root_certificates_bytes is None),
-            root_certificates=self.root_certificates_bytes,
-        )
-        self.stub = DriverStub(channel)
-
     def _create_run(
         self,
         fab: Fab,
         override_config: UserConfig,
     ) -> int:
-        if self.stub is None:
-            self._connect()
+        fab_hash = self.ffs.put(fab.content, {})
+        if fab_hash != fab.hash_str:
+            raise RuntimeError(
+                f"FAB ({fab.hash_str}) hash from request doesn't match contents"
+            )
 
-        assert self.stub is not None
+        run_id = self.linkstate.create_run(
+            None, None, fab_hash, override_config, ConfigsRecord()
+        )
+        return run_id
 
-        req = CreateRunRequest(
-            fab=fab_to_proto(fab),
-            override_config=user_config_to_proto(override_config),
+    def _create_context(self, run_id: int) -> None:
+        """Register a Context for a Run."""
+        # Create an empty context for the Run
+        context = Context(
+            run_id=run_id, node_id=0, node_config={}, state=RecordSet(), run_config={}
         )
-        res = self.stub.CreateRun(request=req)
-        return int(res.run_id)
+
+        # Register the context at the LinkState
+        self.linkstate.set_serverapp_context(run_id=run_id, context=context)
 
     @override
     def start_run(
         self,
         fab_file: bytes,
         override_config: UserConfig,
-        federation_config: UserConfig,
-    ) -> Optional[RunTracker]:
+        federation_options: ConfigsRecord,
+    ) -> Optional[int]:
         """Start run using the Flower Deployment Engine."""
+        run_id = None
         try:
-            # Install FAB to flwr dir
-            install_from_fab(fab_file, None, True)
 
             # Call SuperLink to create run
-            run_id: int = self._create_run(
+            run_id = self._create_run(
                 Fab(hashlib.sha256(fab_file).hexdigest(), fab_file), override_config
             )
-            log(INFO, "Created run %s", str(run_id))
 
-            command = [
-                "flower-server-app",
-                "--run-id",
-                str(run_id),
-                "--superlink",
-                str(self.superlink),
-            ]
-
-            if self.flwr_dir:
-                command.append("--flwr-dir")
-                command.append(self.flwr_dir)
-
-            if self.root_certificates is None:
-                command.append("--insecure")
-            else:
-                command.append("--root-certificates")
-                command.append(self.root_certificates)
-
-            # Execute the command
-            proc = subprocess.Popen(  # pylint: disable=consider-using-with
-                command,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                text=True,
-            )
-            log(INFO, "Started run %s", str(run_id))
+            # Register context for the Run
+            self._create_context(run_id=run_id)
+            log(INFO, "Created run %s", str(run_id))
 
-            return RunTracker(
-                run_id=run_id,
-                proc=proc,
-            )
+            return run_id
         # pylint: disable-next=broad-except
         except Exception as e:
             log(ERROR, "Could not start run: %s", str(e))
+            if run_id:
+                run_status = RunStatus(Status.FINISHED, SubStatus.FAILED, str(e))
+                self.linkstate.update_run_status(run_id, new_status=run_status)
             return None
 
 

flwr/superexec/exec_grpc.py

@@ -23,33 +23,40 @@ from flwr.common import GRPC_MAX_MESSAGE_LENGTH
 from flwr.common.logger import log
 from flwr.common.typing import UserConfig
 from flwr.proto.exec_pb2_grpc import add_ExecServicer_to_server
+from flwr.server.superlink.ffs.ffs_factory import FfsFactory
 from flwr.server.superlink.fleet.grpc_bidi.grpc_server import generic_create_grpc_server
+from flwr.server.superlink.linkstate import LinkStateFactory
 
 from .exec_servicer import ExecServicer
 from .executor import Executor
 
 
-def run_superexec_api_grpc(
+# pylint: disable-next=too-many-arguments, too-many-positional-arguments
+def run_exec_api_grpc(
     address: str,
     executor: Executor,
+    state_factory: LinkStateFactory,
+    ffs_factory: FfsFactory,
     certificates: Optional[tuple[bytes, bytes, bytes]],
     config: UserConfig,
 ) -> grpc.Server:
-    """Run SuperExec API (gRPC, request-response)."""
+    """Run Exec API (gRPC, request-response)."""
     executor.set_config(config)
 
     exec_servicer: grpc.Server = ExecServicer(
+        linkstate_factory=state_factory,
+        ffs_factory=ffs_factory,
         executor=executor,
     )
-    superexec_add_servicer_to_server_fn = add_ExecServicer_to_server
-    superexec_grpc_server = generic_create_grpc_server(
-        servicer_and_add_fn=(exec_servicer, superexec_add_servicer_to_server_fn),
+    exec_add_servicer_to_server_fn = add_ExecServicer_to_server
+    exec_grpc_server = generic_create_grpc_server(
+        servicer_and_add_fn=(exec_servicer, exec_add_servicer_to_server_fn),
         server_address=address,
         max_message_length=GRPC_MAX_MESSAGE_LENGTH,
         certificates=certificates,
     )
 
-    log(INFO, "Starting Flower SuperExec gRPC server on %s", address)
-    superexec_grpc_server.start()
+    log(INFO, "Flower Deployment Engine: Starting Exec API on %s", address)
+    exec_grpc_server.start()
 
-    return superexec_grpc_server
+    return exec_grpc_server

flwr/superexec/exec_servicer.py

@@ -15,9 +15,6 @@
 """SuperExec API servicer."""
 
 
-import select
-import sys
-import threading
 import time
 from collections.abc import Generator
 from logging import ERROR, INFO
@@ -25,8 +22,9 @@ from typing import Any
 
 import grpc
 
+from flwr.common.constant import LOG_STREAM_INTERVAL, Status
 from flwr.common.logger import log
-from flwr.common.serde import user_config_from_proto
+from flwr.common.serde import configs_record_from_proto, user_config_from_proto
 from flwr.proto import exec_pb2_grpc  # pylint: disable=E0611
 from flwr.proto.exec_pb2 import (  # pylint: disable=E0611
     StartRunRequest,
@@ -34,18 +32,25 @@ from flwr.proto.exec_pb2 import (  # pylint: disable=E0611
     StreamLogsRequest,
     StreamLogsResponse,
 )
+from flwr.server.superlink.ffs.ffs_factory import FfsFactory
+from flwr.server.superlink.linkstate import LinkStateFactory
 
-from .executor import Executor, RunTracker
-
-SELECT_TIMEOUT = 1  # Timeout for selecting ready-to-read file descriptors (in seconds)
+from .executor import Executor
 
 
 class ExecServicer(exec_pb2_grpc.ExecServicer):
     """SuperExec API servicer."""
 
-    def __init__(self, executor: Executor) -> None:
+    def __init__(
+        self,
+        linkstate_factory: LinkStateFactory,
+        ffs_factory: FfsFactory,
+        executor: Executor,
+    ) -> None:
+        self.linkstate_factory = linkstate_factory
+        self.ffs_factory = ffs_factory
         self.executor = executor
-        self.runs: dict[int, RunTracker] = {}
+        self.executor.initialize(linkstate_factory, ffs_factory)
 
     def StartRun(
         self, request: StartRunRequest, context: grpc.ServicerContext
@@ -53,84 +58,50 @@ class ExecServicer(exec_pb2_grpc.ExecServicer):
         """Create run ID."""
         log(INFO, "ExecServicer.StartRun")
 
-        run = self.executor.start_run(
+        run_id = self.executor.start_run(
             request.fab.content,
             user_config_from_proto(request.override_config),
-            user_config_from_proto(request.federation_config),
+            configs_record_from_proto(request.federation_options),
         )
 
-        if run is None:
+        if run_id is None:
             log(ERROR, "Executor failed to start run")
             return StartRunResponse()
 
-        self.runs[run.run_id] = run
-
-        # Start a background thread to capture the log output
-        capture_thread = threading.Thread(
-            target=_capture_logs, args=(run,), daemon=True
-        )
-        capture_thread.start()
-
-        return StartRunResponse(run_id=run.run_id)
+        return StartRunResponse(run_id=run_id)
 
     def StreamLogs(  # pylint: disable=C0103
         self, request: StreamLogsRequest, context: grpc.ServicerContext
     ) -> Generator[StreamLogsResponse, Any, None]:
         """Get logs."""
         log(INFO, "ExecServicer.StreamLogs")
+        state = self.linkstate_factory.state()
+
+        # Retrieve run ID
+        run_id = request.run_id
 
         # Exit if `run_id` not found
-        if request.run_id not in self.runs:
+        if not state.get_run(run_id):
             context.abort(grpc.StatusCode.NOT_FOUND, "Run ID not found")
 
-        last_sent_index = 0
+        after_timestamp = request.after_timestamp + 1e-6
         while context.is_active():
-            # Yield n'th row of logs, if n'th row < len(logs)
-            logs = self.runs[request.run_id].logs
-            for i in range(last_sent_index, len(logs)):
-                yield StreamLogsResponse(log_output=logs[i])
-            last_sent_index = len(logs)
+            log_msg, latest_timestamp = state.get_serverapp_log(run_id, after_timestamp)
+            if log_msg:
+                yield StreamLogsResponse(
+                    log_output=log_msg,
+                    latest_timestamp=latest_timestamp,
+                )
+                # Add a small epsilon to the latest timestamp to avoid getting
+                # the same log
+                after_timestamp = max(latest_timestamp + 1e-6, after_timestamp)
 
             # Wait for and continue to yield more log responses only if the
             # run isn't completed yet. If the run is finished, the entire log
             # is returned at this point and the server ends the stream.
-            if self.runs[request.run_id].proc.poll() is not None:
+            run_status = state.get_run_status({run_id})[run_id]
+            if run_status.status == Status.FINISHED:
                 log(INFO, "All logs for run ID `%s` returned", request.run_id)
-                context.set_code(grpc.StatusCode.OK)
                 context.cancel()
 
-            time.sleep(1.0)  # Sleep briefly to avoid busy waiting
-
-
-def _capture_logs(
-    run: RunTracker,
-) -> None:
-    while True:
-        # Explicitly check if Popen.poll() is None. Required for `pytest`.
-        if run.proc.poll() is None:
-            # Select streams only when ready to read
-            ready_to_read, _, _ = select.select(
-                [run.proc.stdout, run.proc.stderr],
-                [],
-                [],
-                SELECT_TIMEOUT,
-            )
-            # Read from std* and append to RunTracker.logs
-            for stream in ready_to_read:
-                # Flush stdout to view output in real time
-                readline = stream.readline()
-                sys.stdout.write(readline)
-                sys.stdout.flush()
-                # Append to logs
-                line = readline.rstrip()
-                if line:
-                    run.logs.append(f"{line}")
-
-        # Close std* to prevent blocking
-        elif run.proc.poll() is not None:
-            log(INFO, "Subprocess finished, exiting log capture")
-            if run.proc.stdout:
-                run.proc.stdout.close()
-            if run.proc.stderr:
-                run.proc.stderr.close()
-            break
+            time.sleep(LOG_STREAM_INTERVAL)  # Sleep briefly to avoid busy waiting

flwr/superexec/executor.py

@@ -19,7 +19,10 @@ from dataclasses import dataclass, field
 from subprocess import Popen
 from typing import Optional
 
+from flwr.common import ConfigsRecord
 from flwr.common.typing import UserConfig
+from flwr.server.superlink.ffs.ffs_factory import FfsFactory
+from flwr.server.superlink.linkstate import LinkStateFactory
 
 
 @dataclass
@@ -34,6 +37,23 @@ class RunTracker:
 class Executor(ABC):
     """Execute and monitor a Flower run."""
 
+    @abstractmethod
+    def initialize(
+        self, linkstate_factory: LinkStateFactory, ffs_factory: FfsFactory
+    ) -> None:
+        """Initialize the executor with the necessary factories.
+
+        This method sets up the executor by providing it with the factories required
+        to access the LinkState and the Flower File Storage (FFS) in the SuperLink.
+
+        Parameters
+        ----------
+        linkstate_factory : LinkStateFactory
+            The factory to create access to the LinkState.
+        ffs_factory : FfsFactory
+            The factory to create access to the Flower File Storage (FFS).
+        """
+
     @abstractmethod
     def set_config(
         self,
@@ -52,8 +72,8 @@ class Executor(ABC):
         self,
         fab_file: bytes,
         override_config: UserConfig,
-        federation_config: UserConfig,
-    ) -> Optional[RunTracker]:
+        federation_options: ConfigsRecord,
+    ) -> Optional[int]:
         """Start a run using the given Flower FAB ID and version.
 
         This method creates a new run on the SuperLink, returns its run_id
@@ -65,12 +85,11 @@ class Executor(ABC):
             The Flower App Bundle file bytes.
         override_config: UserConfig
             The config overrides dict sent by the user (using `flwr run`).
-        federation_config: UserConfig
-            The federation options dict sent by the user (using `flwr run`).
+        federation_options: ConfigsRecord
+            The federation options sent by the user (using `flwr run`).
 
         Returns
         -------
-        run_id : Optional[RunTracker]
-            The run_id and the associated process of the run created by the SuperLink,
-            or `None` if it fails.
+        run_id : Optional[int]
+            The run_id of the run created by the SuperLink, or `None` if it fails.
         """

flwr/superexec/simulation.py

@@ -15,42 +15,20 @@
 """Simulation engine executor."""
 
 
-import json
-import subprocess
-import sys
-from logging import ERROR, INFO, WARN
+import hashlib
+from logging import ERROR, INFO
 from typing import Optional
 
 from typing_extensions import override
 
-from flwr.cli.config_utils import load_and_validate
-from flwr.cli.install import install_from_fab
-from flwr.common.config import unflatten_dict
-from flwr.common.constant import RUN_ID_NUM_BYTES
+from flwr.common import ConfigsRecord, Context, RecordSet
 from flwr.common.logger import log
-from flwr.common.typing import UserConfig
-from flwr.server.superlink.state.utils import generate_rand_int_from_bytes
-
-from .executor import Executor, RunTracker
-
-
-def _user_config_to_str(user_config: UserConfig) -> str:
-    """Convert override user config to string."""
-    user_config_list_str = []
-    for key, value in user_config.items():
-        if isinstance(value, bool):
-            user_config_list_str.append(f"{key}={str(value).lower()}")
-        elif isinstance(value, (int, float)):
-            user_config_list_str.append(f"{key}={value}")
-        elif isinstance(value, str):
-            user_config_list_str.append(f'{key}="{value}"')
-        else:
-            raise ValueError(
-                "Only types `bool`, `float`, `int` and `str` are supported"
-            )
+from flwr.common.typing import Fab, UserConfig
+from flwr.server.superlink.ffs import Ffs
+from flwr.server.superlink.ffs.ffs_factory import FfsFactory
+from flwr.server.superlink.linkstate import LinkState, LinkStateFactory
 
-    user_config_str = ",".join(user_config_list_str)
-    return user_config_str
+from .executor import Executor
 
 
 class SimulationEngine(Executor):
@@ -69,6 +47,30 @@ class SimulationEngine(Executor):
     ) -> None:
         self.num_supernodes = num_supernodes
         self.verbose = verbose
+        self.linkstate_factory: Optional[LinkStateFactory] = None
+        self.ffs_factory: Optional[FfsFactory] = None
+
+    @override
+    def initialize(
+        self, linkstate_factory: LinkStateFactory, ffs_factory: FfsFactory
+    ) -> None:
+        """Initialize the executor with the necessary factories."""
+        self.linkstate_factory = linkstate_factory
+        self.ffs_factory = ffs_factory
+
+    @property
+    def linkstate(self) -> LinkState:
+        """Return the LinkState."""
+        if self.linkstate_factory is None:
+            raise RuntimeError("Executor is not initialized.")
+        return self.linkstate_factory.state()
+
+    @property
+    def ffs(self) -> Ffs:
+        """Return the Flower File Storage (FFS)."""
+        if self.ffs_factory is None:
+            raise RuntimeError("Executor is not initialized.")
+        return self.ffs_factory.ffs()
 
     @override
     def set_config(
@@ -116,92 +118,37 @@ class SimulationEngine(Executor):
         self,
         fab_file: bytes,
         override_config: UserConfig,
-        federation_config: UserConfig,
-    ) -> Optional[RunTracker]:
+        federation_options: ConfigsRecord,
+    ) -> Optional[int]:
         """Start run using the Flower Simulation Engine."""
-        if self.num_supernodes is None:
-            raise ValueError(
-                "Error in `SuperExec` (`SimulationEngine` executor):\n\n"
-                "`num-supernodes` must not be `None`, it must be a valid "
-                "positive integer. In order to start this simulation executor "
-                "with a specified number of `SuperNodes`, you can either provide "
-                "a `--executor` that has been initialized with a number of nodes "
-                "to the `flower-superexec` CLI, or `--executor-config num-supernodes=N`"
-                "to the `flower-superexec` CLI."
-            )
         try:
-
-            # Install FAB to flwr dir
-            fab_path = install_from_fab(fab_file, None, True)
-
-            # Install FAB Python package
-            subprocess.run(
-                [sys.executable, "-m", "pip", "install", "--no-deps", str(fab_path)],
-                stdout=None if self.verbose else subprocess.DEVNULL,
-                stderr=None if self.verbose else subprocess.DEVNULL,
-                check=True,
-            )
-
-            # Load and validate config
-            config, errors, warnings = load_and_validate(fab_path / "pyproject.toml")
-            if errors:
-                raise ValueError(errors)
-
-            if warnings:
-                log(WARN, warnings)
-
-            if config is None:
-                raise ValueError(
-                    "Config extracted from FAB's pyproject.toml is not valid"
+            # Create run
+            fab = Fab(hashlib.sha256(fab_file).hexdigest(), fab_file)
+            fab_hash = self.ffs.put(fab.content, {})
+            if fab_hash != fab.hash_str:
+                raise RuntimeError(
+                    f"FAB ({fab.hash_str}) hash from request doesn't match contents"
                 )
 
-            # Flatten federated config
-            federation_config_flat = unflatten_dict(federation_config)
-
-            num_supernodes = federation_config_flat.get(
-                "num-supernodes", self.num_supernodes
+            run_id = self.linkstate.create_run(
+                None, None, fab_hash, override_config, federation_options
             )
-            backend_cfg = federation_config_flat.get("backend", {})
-            verbose: Optional[bool] = federation_config_flat.get("verbose")
-
-            # In Simulation there is no SuperLink, still we create a run_id
-            run_id = generate_rand_int_from_bytes(RUN_ID_NUM_BYTES)
-            log(INFO, "Created run %s", str(run_id))
 
-            # Prepare commnand
-            command = [
-                "flower-simulation",
-                "--app",
-                f"{str(fab_path)}",
-                "--num-supernodes",
-                f"{num_supernodes}",
-                "--run-id",
-                str(run_id),
-            ]
-
-            if backend_cfg:
-                # Stringify as JSON
-                command.extend(["--backend-config", json.dumps(backend_cfg)])
-
-            if verbose:
-                command.extend(["--verbose"])
-
-            if override_config:
-                override_config_str = _user_config_to_str(override_config)
-                command.extend(["--run-config", f"{override_config_str}"])
-
-            # Start Simulation
-            proc = subprocess.Popen(  # pylint: disable=consider-using-with
-                command,
-                text=True,
+            # Create an empty context for the Run
+            context = Context(
+                run_id=run_id,
+                node_id=0,
+                node_config={},
+                state=RecordSet(),
+                run_config={},
             )
 
-            log(INFO, "Started run %s", str(run_id))
+            # Register the context at the LinkState
+            self.linkstate.set_serverapp_context(run_id=run_id, context=context)
 
-            return RunTracker(
-                run_id=run_id,
-                proc=proc,
-            )
+            log(INFO, "Created run %s", str(run_id))
+
+            return run_id
 
         # pylint: disable-next=broad-except
         except Exception as e: