flwr-nightly 1.13.0.dev20241028__py3-none-any.whl → 1.13.0.dev20241029__py3-none-any.whl

flwr/server/serverapp/app.py

@@ -16,13 +16,38 @@
 
 import argparse
 import sys
-from logging import DEBUG, INFO, WARN
+from logging import DEBUG, ERROR, INFO, WARN
 from os.path import isfile
 from pathlib import Path
+from time import sleep
 from typing import Optional
 
+from flwr.cli.config_utils import get_fab_metadata
+from flwr.cli.install import install_from_fab
+from flwr.common.config import (
+    get_flwr_dir,
+    get_fused_config_from_dir,
+    get_project_config,
+    get_project_dir,
+)
+from flwr.common.constant import Status, SubStatus
 from flwr.common.logger import log
+from flwr.common.serde import (
+    context_from_proto,
+    context_to_proto,
+    fab_from_proto,
+    run_from_proto,
+    run_status_to_proto,
+)
+from flwr.common.typing import RunStatus
+from flwr.proto.driver_pb2 import (  # pylint: disable=E0611
+    PullServerAppInputsRequest,
+    PullServerAppInputsResponse,
+    PushServerAppOutputsRequest,
+)
+from flwr.proto.run_pb2 import UpdateRunStatusRequest  # pylint: disable=E0611
 from flwr.server.driver.grpc_driver import GrpcDriver
+from flwr.server.run_serverapp import run as run_
 
 
 def flwr_serverapp() -> None:
@@ -121,21 +146,108 @@ def _try_obtain_certificates(
     return root_certificates
 
 
-def run_serverapp(  # pylint: disable=R0914
+def run_serverapp(  # pylint: disable=R0914, disable=W0212
     superlink: str,
     run_id: Optional[int] = None,
     flwr_dir_: Optional[str] = None,
     certificates: Optional[bytes] = None,
 ) -> None:
     """Run Flower ServerApp process."""
-    _ = GrpcDriver(
-        run_id=run_id if run_id else 0,
+    driver = GrpcDriver(
         driver_service_address=superlink,
         root_certificates=certificates,
     )
 
-    log(INFO, "%s", flwr_dir_)
+    # Resolve directory where FABs are installed
+    flwr_dir = get_flwr_dir(flwr_dir_)
 
-    # Then, GetServerInputs
+    only_once = run_id is not None
 
-    # Then, run ServerApp
+    while True:
+
+        try:
+            # Pull ServerAppInputs from LinkState
+            req = (
+                PullServerAppInputsRequest(run_id=run_id)
+                if run_id
+                else PullServerAppInputsRequest()
+            )
+            res: PullServerAppInputsResponse = driver._stub.PullServerAppInputs(req)
+            if not res.HasField("run"):
+                sleep(3)
+                run_status = None
+                continue
+
+            context = context_from_proto(res.context)
+            run = run_from_proto(res.run)
+            fab = fab_from_proto(res.fab)
+
+            driver.init_run(run.run_id)
+
+            log(DEBUG, "ServerApp process starts FAB installation.")
+            install_from_fab(fab.content, flwr_dir=flwr_dir, skip_prompt=True)
+
+            fab_id, fab_version = get_fab_metadata(fab.content)
+
+            app_path = str(get_project_dir(fab_id, fab_version, fab.hash_str, flwr_dir))
+            config = get_project_config(app_path)
+
+            # Obtain server app reference and the run config
+            server_app_attr = config["tool"]["flwr"]["app"]["components"]["serverapp"]
+            server_app_run_config = get_fused_config_from_dir(
+                Path(app_path), run.override_config
+            )
+
+            # Update run_config in context
+            context.run_config = server_app_run_config
+
+            log(
+                DEBUG,
+                "Flower will load ServerApp `%s` in %s",
+                server_app_attr,
+                app_path,
+            )
+
+            # Change status to Running
+            run_status_proto = run_status_to_proto(RunStatus(Status.RUNNING, "", ""))
+            driver._stub.UpdateRunStatus(
+                UpdateRunStatusRequest(run_id=run.run_id, run_status=run_status_proto)
+            )
+
+            # Load and run the ServerApp with the Driver
+            updated_context = run_(
+                driver=driver,
+                server_app_dir=app_path,
+                server_app_attr=server_app_attr,
+                context=context,
+            )
+
+            # Send resulting context
+            context_proto = context_to_proto(updated_context)
+            out_req = PushServerAppOutputsRequest(
+                run_id=run.run_id, context=context_proto
+            )
+            _ = driver._stub.PushServerAppOutputs(out_req)
+
+            run_status = RunStatus(Status.FINISHED, SubStatus.COMPLETED, "")
+
+        except Exception as ex:  # pylint: disable=broad-exception-caught
+            exc_entity = "ServerApp"
+            log(ERROR, "%s raised an exception", exc_entity, exc_info=ex)
+            run_status = RunStatus(Status.FINISHED, SubStatus.FAILED, str(ex))
+
+        finally:
+            if run_status:
+                run_status_proto = run_status_to_proto(run_status)
+                driver._stub.UpdateRunStatus(
+                    UpdateRunStatusRequest(
+                        run_id=run.run_id, run_status=run_status_proto
+                    )
+                )
+
+        # Stop the loop if `flwr-serverapp` is expected to process a single run
+        if only_once:
+            break
+
+        # Reset the run_id
+        run_id = None

flwr/server/superlink/driver/driver_servicer.py

@@ -31,6 +31,7 @@ from flwr.common.serde import (
     context_to_proto,
     fab_from_proto,
     fab_to_proto,
+    run_status_from_proto,
     run_to_proto,
     user_config_from_proto,
 )
@@ -49,12 +50,18 @@ from flwr.proto.driver_pb2 import (  # pylint: disable=E0611
     PushTaskInsResponse,
 )
 from flwr.proto.fab_pb2 import GetFabRequest, GetFabResponse  # pylint: disable=E0611
+from flwr.proto.log_pb2 import (  # pylint: disable=E0611
+    PushLogsRequest,
+    PushLogsResponse,
+)
 from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
 from flwr.proto.run_pb2 import (  # pylint: disable=E0611
     CreateRunRequest,
     CreateRunResponse,
     GetRunRequest,
     GetRunResponse,
+    UpdateRunStatusRequest,
+    UpdateRunStatusResponse,
 )
 from flwr.proto.task_pb2 import TaskRes  # pylint: disable=E0611
 from flwr.server.superlink.ffs.ffs import Ffs
@@ -253,6 +260,26 @@ class DriverServicer(driver_pb2_grpc.DriverServicer):
         state.set_serverapp_context(request.run_id, context_from_proto(request.context))
         return PushServerAppOutputsResponse()
 
+    def UpdateRunStatus(
+        self, request: UpdateRunStatusRequest, context: grpc.ServicerContext
+    ) -> UpdateRunStatusResponse:
+        """Update the status of a run."""
+        log(DEBUG, "ControlServicer.UpdateRunStatus")
+        state = self.state_factory.state()
+
+        # Update the run status
+        state.update_run_status(
+            run_id=request.run_id, new_status=run_status_from_proto(request.run_status)
+        )
+        return UpdateRunStatusResponse()
+
+    def PushLogs(
+        self, request: PushLogsRequest, context: grpc.ServicerContext
+    ) -> PushLogsResponse:
+        """Push logs."""
+        log(DEBUG, "DriverServicer.PushLogs")
+        raise NotImplementedError()
+
 
 def _raise_if(validation_error: bool, detail: str) -> None:
     if validation_error:

flwr/server/superlink/linkstate/in_memory_linkstate.py

@@ -17,7 +17,8 @@
 
 import threading
 import time
-from dataclasses import dataclass
+from bisect import bisect_right
+from dataclasses import dataclass, field
 from logging import ERROR, WARNING
 from typing import Optional
 from uuid import UUID, uuid4
@@ -43,7 +44,7 @@ from .utils import (
 
 
 @dataclass
-class RunRecord:
+class RunRecord:  # pylint: disable=R0902
     """The record of a specific run, including its status and timestamps."""
 
     run: Run
@@ -52,6 +53,8 @@ class RunRecord:
     starting_at: str = ""
     running_at: str = ""
     finished_at: str = ""
+    logs: list[tuple[float, str]] = field(default_factory=list)
+    log_lock: threading.Lock = field(default_factory=threading.Lock)
 
 
 class InMemoryLinkState(LinkState):  # pylint: disable=R0902,R0904
@@ -511,3 +514,26 @@ class InMemoryLinkState(LinkState):  # pylint: disable=R0902,R0904
         if run_id not in self.run_ids:
             raise ValueError(f"Run {run_id} not found")
         self.contexts[run_id] = context
+
+    def add_serverapp_log(self, run_id: int, log_message: str) -> None:
+        """Add a log entry to the serverapp logs for the specified `run_id`."""
+        if run_id not in self.run_ids:
+            raise ValueError(f"Run {run_id} not found")
+        run = self.run_ids[run_id]
+        with run.log_lock:
+            run.logs.append((now().timestamp(), log_message))
+
+    def get_serverapp_log(
+        self, run_id: int, after_timestamp: Optional[float]
+    ) -> tuple[str, float]:
+        """Get the serverapp logs for the specified `run_id`."""
+        if run_id not in self.run_ids:
+            raise ValueError(f"Run {run_id} not found")
+        run = self.run_ids[run_id]
+        if after_timestamp is None:
+            after_timestamp = 0.0
+        with run.log_lock:
+            # Find the index where the timestamp would be inserted
+            index = bisect_right(run.logs, (after_timestamp, ""))
+            latest_timestamp = run.logs[-1][0] if index < len(run.logs) else 0.0
+            return "".join(log for _, log in run.logs[index:]), latest_timestamp

flwr/server/superlink/linkstate/linkstate.py

@@ -299,3 +299,38 @@ class LinkState(abc.ABC):  # pylint: disable=R0904
         context : Context
             The context to be associated with the specified `run_id`.
         """
+
+    @abc.abstractmethod
+    def add_serverapp_log(self, run_id: int, log_message: str) -> None:
+        """Add a log entry to the ServerApp logs for the specified `run_id`.
+
+        Parameters
+        ----------
+        run_id : int
+            The identifier of the run for which to add a log entry.
+        log_message : str
+            The log entry to be added to the ServerApp logs.
+        """
+
+    @abc.abstractmethod
+    def get_serverapp_log(
+        self, run_id: int, after_timestamp: Optional[float]
+    ) -> tuple[str, float]:
+        """Get the ServerApp logs for the specified `run_id`.
+
+        Parameters
+        ----------
+        run_id : int
+            The identifier of the run for which to retrieve the ServerApp logs.
+
+        after_timestamp : Optional[float]
+            Retrieve logs after this timestamp. If set to `None`, retrieve all logs.
+
+        Returns
+        -------
+        tuple[str, float]
+            A tuple containing:
+            - The ServerApp logs associated with the specified `run_id`.
+            - The timestamp of the latest log entry in the returned logs.
+              Returns `0` if no logs are returned.
+        """

flwr/server/superlink/linkstate/sqlite_linkstate.py

@@ -99,6 +99,17 @@ CREATE TABLE IF NOT EXISTS run(
 );
 """
 
+SQL_CREATE_TABLE_LOGS = """
+CREATE TABLE IF NOT EXISTS logs (
+    timestamp             REAL,
+    run_id                INTEGER,
+    node_id               INTEGER,
+    log                   TEXT,
+    PRIMARY KEY (timestamp, run_id, node_id),
+    FOREIGN KEY (run_id) REFERENCES run(run_id)
+);
+"""
+
 SQL_CREATE_TABLE_CONTEXT = """
 CREATE TABLE IF NOT EXISTS context(
     run_id                INTEGER UNIQUE,
@@ -191,6 +202,7 @@ class SqliteLinkState(LinkState):  # pylint: disable=R0904
 
         # Create each table if not exists queries
         cur.execute(SQL_CREATE_TABLE_RUN)
+        cur.execute(SQL_CREATE_TABLE_LOGS)
         cur.execute(SQL_CREATE_TABLE_CONTEXT)
         cur.execute(SQL_CREATE_TABLE_TASK_INS)
         cur.execute(SQL_CREATE_TABLE_TASK_RES)
@@ -1015,6 +1027,44 @@ class SqliteLinkState(LinkState):  # pylint: disable=R0904
             except sqlite3.IntegrityError:
                 raise ValueError(f"Run {run_id} not found") from None
 
+    def add_serverapp_log(self, run_id: int, log_message: str) -> None:
+        """Add a log entry to the ServerApp logs for the specified `run_id`."""
+        # Convert the uint64 value to sint64 for SQLite
+        sint64_run_id = convert_uint64_to_sint64(run_id)
+
+        # Store log
+        try:
+            query = """
+                INSERT INTO logs (timestamp, run_id, node_id, log) VALUES (?, ?, ?, ?);
+            """
+            self.query(query, (now().timestamp(), sint64_run_id, 0, log_message))
+        except sqlite3.IntegrityError:
+            raise ValueError(f"Run {run_id} not found") from None
+
+    def get_serverapp_log(
+        self, run_id: int, after_timestamp: Optional[float]
+    ) -> tuple[str, float]:
+        """Get the ServerApp logs for the specified `run_id`."""
+        # Convert the uint64 value to sint64 for SQLite
+        sint64_run_id = convert_uint64_to_sint64(run_id)
+
+        # Check if the run_id exists
+        query = "SELECT run_id FROM run WHERE run_id = ?;"
+        if not self.query(query, (sint64_run_id,)):
+            raise ValueError(f"Run {run_id} not found")
+
+        # Retrieve logs
+        if after_timestamp is None:
+            after_timestamp = 0.0
+        query = """
+            SELECT log, timestamp FROM logs
+            WHERE run_id = ? AND node_id = ? AND timestamp > ?;
+        """
+        rows = self.query(query, (sint64_run_id, 0, after_timestamp))
+        rows.sort(key=lambda x: x["timestamp"])
+        latest_timestamp = rows[-1]["timestamp"] if rows else 0.0
+        return "".join(row["log"] for row in rows), latest_timestamp
+
     def get_valid_task_ins(self, task_id: str) -> Optional[dict[str, Any]]:
         """Check if the TaskIns exists and is valid (not expired).
 

flwr/simulation/run_simulation.py

@@ -421,7 +421,8 @@ def _main_loop(
             server_app_run_config = {}
 
         # Initialize Driver
-        driver = InMemoryDriver(run_id=run.run_id, state_factory=state_factory)
+        driver = InMemoryDriver(state_factory=state_factory)
+        driver.init_run(run_id=run.run_id)
 
         # Get and run ServerApp thread
         serverapp_th = run_serverapp_th(

flwr/superexec/deployment.py

@@ -15,14 +15,12 @@
 """Deployment engine executor."""
 
 import hashlib
-import subprocess
 from logging import ERROR, INFO
 from pathlib import Path
 from typing import Optional
 
 from typing_extensions import override
 
-from flwr.cli.install import install_from_fab
 from flwr.common.constant import DRIVER_API_DEFAULT_ADDRESS
 from flwr.common.logger import log
 from flwr.common.typing import Fab, UserConfig
@@ -30,7 +28,7 @@ from flwr.server.superlink.ffs import Ffs
 from flwr.server.superlink.ffs.ffs_factory import FfsFactory
 from flwr.server.superlink.linkstate import LinkState, LinkStateFactory
 
-from .executor import Executor, RunTracker
+from .executor import Executor
 
 
 class DeploymentEngine(Executor):
@@ -143,11 +141,9 @@ class DeploymentEngine(Executor):
         fab_file: bytes,
         override_config: UserConfig,
         federation_config: UserConfig,
-    ) -> Optional[RunTracker]:
+    ) -> Optional[int]:
         """Start run using the Flower Deployment Engine."""
         try:
-            # Install FAB to flwr dir
-            install_from_fab(fab_file, None, True)
 
             # Call SuperLink to create run
             run_id: int = self._create_run(
@@ -155,37 +151,7 @@ class DeploymentEngine(Executor):
             )
             log(INFO, "Created run %s", str(run_id))
 
-            command = [
-                "flower-server-app",
-                "--run-id",
-                str(run_id),
-                "--superlink",
-                str(self.superlink),
-            ]
-
-            if self.flwr_dir:
-                command.append("--flwr-dir")
-                command.append(self.flwr_dir)
-
-            if self.root_certificates is None:
-                command.append("--insecure")
-            else:
-                command.append("--root-certificates")
-                command.append(self.root_certificates)
-
-            # Execute the command
-            proc = subprocess.Popen(  # pylint: disable=consider-using-with
-                command,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                text=True,
-            )
-            log(INFO, "Started run %s", str(run_id))
-
-            return RunTracker(
-                run_id=run_id,
-                proc=proc,
-            )
+            return run_id
         # pylint: disable-next=broad-except
         except Exception as e:
             log(ERROR, "Could not start run: %s", str(e))

flwr/superexec/exec_servicer.py

@@ -15,10 +15,6 @@
 """SuperExec API servicer."""
 
 
-import select
-import sys
-import threading
-import time
 from collections.abc import Generator
 from logging import ERROR, INFO
 from typing import Any
@@ -37,7 +33,7 @@ from flwr.proto.exec_pb2 import (  # pylint: disable=E0611
 from flwr.server.superlink.ffs.ffs_factory import FfsFactory
 from flwr.server.superlink.linkstate import LinkStateFactory
 
-from .executor import Executor, RunTracker
+from .executor import Executor
 
 SELECT_TIMEOUT = 1  # Timeout for selecting ready-to-read file descriptors (in seconds)
 
@@ -55,7 +51,6 @@ class ExecServicer(exec_pb2_grpc.ExecServicer):
         self.ffs_factory = ffs_factory
         self.executor = executor
         self.executor.initialize(linkstate_factory, ffs_factory)
-        self.runs: dict[int, RunTracker] = {}
 
     def StartRun(
         self, request: StartRunRequest, context: grpc.ServicerContext
@@ -63,25 +58,17 @@ class ExecServicer(exec_pb2_grpc.ExecServicer):
         """Create run ID."""
         log(INFO, "ExecServicer.StartRun")
 
-        run = self.executor.start_run(
+        run_id = self.executor.start_run(
             request.fab.content,
             user_config_from_proto(request.override_config),
             user_config_from_proto(request.federation_config),
         )
 
-        if run is None:
+        if run_id is None:
             log(ERROR, "Executor failed to start run")
             return StartRunResponse()
 
-        self.runs[run.run_id] = run
-
-        # Start a background thread to capture the log output
-        capture_thread = threading.Thread(
-            target=_capture_logs, args=(run,), daemon=True
-        )
-        capture_thread.start()
-
-        return StartRunResponse(run_id=run.run_id)
+        return StartRunResponse(run_id=run_id)
 
     def StreamLogs(  # pylint: disable=C0103
         self, request: StreamLogsRequest, context: grpc.ServicerContext
@@ -89,58 +76,4 @@ class ExecServicer(exec_pb2_grpc.ExecServicer):
         """Get logs."""
         log(INFO, "ExecServicer.StreamLogs")
 
-        # Exit if `run_id` not found
-        if request.run_id not in self.runs:
-            context.abort(grpc.StatusCode.NOT_FOUND, "Run ID not found")
-
-        last_sent_index = 0
-        while context.is_active():
-            # Yield n'th row of logs, if n'th row < len(logs)
-            logs = self.runs[request.run_id].logs
-            for i in range(last_sent_index, len(logs)):
-                yield StreamLogsResponse(log_output=logs[i])
-            last_sent_index = len(logs)
-
-            # Wait for and continue to yield more log responses only if the
-            # run isn't completed yet. If the run is finished, the entire log
-            # is returned at this point and the server ends the stream.
-            if self.runs[request.run_id].proc.poll() is not None:
-                log(INFO, "All logs for run ID `%s` returned", request.run_id)
-                context.set_code(grpc.StatusCode.OK)
-                context.cancel()
-
-            time.sleep(1.0)  # Sleep briefly to avoid busy waiting
-
-
-def _capture_logs(
-    run: RunTracker,
-) -> None:
-    while True:
-        # Explicitly check if Popen.poll() is None. Required for `pytest`.
-        if run.proc.poll() is None:
-            # Select streams only when ready to read
-            ready_to_read, _, _ = select.select(
-                [run.proc.stdout, run.proc.stderr],
-                [],
-                [],
-                SELECT_TIMEOUT,
-            )
-            # Read from std* and append to RunTracker.logs
-            for stream in ready_to_read:
-                # Flush stdout to view output in real time
-                readline = stream.readline()
-                sys.stdout.write(readline)
-                sys.stdout.flush()
-                # Append to logs
-                line = readline.rstrip()
-                if line:
-                    run.logs.append(f"{line}")
-
-        # Close std* to prevent blocking
-        elif run.proc.poll() is not None:
-            log(INFO, "Subprocess finished, exiting log capture")
-            if run.proc.stdout:
-                run.proc.stdout.close()
-            if run.proc.stderr:
-                run.proc.stderr.close()
-            break
+        yield StreamLogsResponse()

flwr/superexec/executor.py

@@ -72,7 +72,7 @@ class Executor(ABC):
         fab_file: bytes,
         override_config: UserConfig,
         federation_config: UserConfig,
-    ) -> Optional[RunTracker]:
+    ) -> Optional[int]:
         """Start a run using the given Flower FAB ID and version.
 
         This method creates a new run on the SuperLink, returns its run_id
@@ -89,7 +89,6 @@ class Executor(ABC):
 
         Returns
         -------
-        run_id : Optional[RunTracker]
-            The run_id and the associated process of the run created by the SuperLink,
-            or `None` if it fails.
+        run_id : Optional[int]
+            The run_id of the run created by the SuperLink, or `None` if it fails.
         """

flwr/superexec/simulation.py

@@ -33,7 +33,7 @@ from flwr.server.superlink.ffs.ffs_factory import FfsFactory
 from flwr.server.superlink.linkstate import LinkStateFactory
 from flwr.server.superlink.linkstate.utils import generate_rand_int_from_bytes
 
-from .executor import Executor, RunTracker
+from .executor import Executor
 
 
 def _user_config_to_str(user_config: UserConfig) -> str:
@@ -125,7 +125,7 @@ class SimulationEngine(Executor):
         fab_file: bytes,
         override_config: UserConfig,
         federation_config: UserConfig,
-    ) -> Optional[RunTracker]:
+    ) -> Optional[int]:
         """Start run using the Flower Simulation Engine."""
         if self.num_supernodes is None:
             raise ValueError(
@@ -199,17 +199,14 @@ class SimulationEngine(Executor):
                 command.extend(["--run-config", f"{override_config_str}"])
 
             # Start Simulation
-            proc = subprocess.Popen(  # pylint: disable=consider-using-with
+            _ = subprocess.Popen(  # pylint: disable=consider-using-with
                 command,
                 text=True,
             )
 
             log(INFO, "Started run %s", str(run_id))
 
-            return RunTracker(
-                run_id=run_id,
-                proc=proc,
-            )
+            return run_id
 
         # pylint: disable-next=broad-except
         except Exception as e:

{flwr_nightly-1.13.0.dev20241028.dist-info → flwr_nightly-1.13.0.dev20241029.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.13.0.dev20241028
+Version: 1.13.0.dev20241029
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0