flwr-nightly 1.13.0.dev20241029__tar.gz → 1.13.0.dev20241030__tar.gz

{flwr_nightly-1.13.0.dev20241029 → flwr_nightly-1.13.0.dev20241030}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.13.0.dev20241029
+Version: 1.13.0.dev20241030
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0

{flwr_nightly-1.13.0.dev20241029 → flwr_nightly-1.13.0.dev20241030}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "flwr-nightly"
-version = "1.13.0.dev20241029"
+version = "1.13.0.dev20241030"
 description = "Flower: A Friendly Federated Learning Framework"
 license = "Apache-2.0"
 authors = ["The Flower Authors <hello@flower.ai>"]

{flwr_nightly-1.13.0.dev20241029 → flwr_nightly-1.13.0.dev20241030}/src/py/flwr/cli/build.py

@@ -81,8 +81,8 @@ def build(
     if not is_valid_project_name(app.name):
         typer.secho(
             f"❌ The project name {app.name} is invalid, "
-            "a valid project name must start with a letter or an underscore, "
-            "and can only contain letters, digits, and underscores.",
+            "a valid project name must start with a letter, "
+            "and can only contain letters, digits, and hyphens.",
             fg=typer.colors.RED,
             bold=True,
         )

{flwr_nightly-1.13.0.dev20241029 → flwr_nightly-1.13.0.dev20241030}/src/py/flwr/cli/log.py

@@ -18,29 +18,30 @@ import sys
 import time
 from logging import DEBUG, ERROR, INFO
 from pathlib import Path
-from typing import Annotated, Optional
+from typing import Annotated, Optional, cast
 
 import grpc
 import typer
 
 from flwr.cli.config_utils import load_and_validate
+from flwr.common.constant import CONN_RECONNECT_INTERVAL, CONN_REFRESH_PERIOD
 from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel
 from flwr.common.logger import log as logger
 from flwr.proto.exec_pb2 import StreamLogsRequest  # pylint: disable=E0611
 from flwr.proto.exec_pb2_grpc import ExecStub
 
-CONN_REFRESH_PERIOD = 60  # Connection refresh period for log streaming (seconds)
-
 
 def start_stream(
     run_id: int, channel: grpc.Channel, refresh_period: int = CONN_REFRESH_PERIOD
 ) -> None:
     """Start log streaming for a given run ID."""
+    stub = ExecStub(channel)
+    after_timestamp = 0.0
     try:
+        logger(INFO, "Starting logstream for run_id `%s`", run_id)
         while True:
-            logger(INFO, "Starting logstream for run_id `%s`", run_id)
-            stream_logs(run_id, channel, refresh_period)
-            time.sleep(2)
+            after_timestamp = stream_logs(run_id, stub, refresh_period, after_timestamp)
+            time.sleep(CONN_RECONNECT_INTERVAL)
             logger(DEBUG, "Reconnecting to logstream")
     except KeyboardInterrupt:
         logger(INFO, "Exiting logstream")
@@ -54,16 +55,44 @@ def start_stream(
         channel.close()
 
 
-def stream_logs(run_id: int, channel: grpc.Channel, duration: int) -> None:
-    """Stream logs from the beginning of a run with connection refresh."""
-    start_time = time.time()
-    stub = ExecStub(channel)
-    req = StreamLogsRequest(run_id=run_id)
+def stream_logs(
+    run_id: int, stub: ExecStub, duration: int, after_timestamp: float
+) -> float:
+    """Stream logs from the beginning of a run with connection refresh.
+
+    Parameters
+    ----------
+    run_id : int
+        The identifier of the run.
+    stub : ExecStub
+        The gRPC stub to interact with the Exec service.
+    duration : int
+        The timeout duration for each stream connection in seconds.
+    after_timestamp : float
+        The timestamp to start streaming logs from.
+
+    Returns
+    -------
+    float
+        The latest timestamp from the streamed logs or the provided `after_timestamp`
+        if no logs are returned.
+    """
+    req = StreamLogsRequest(run_id=run_id, after_timestamp=after_timestamp)
+
+    latest_timestamp = 0.0
+    res = None
+    try:
+        for res in stub.StreamLogs(req, timeout=duration):
+            print(res.log_output, end="")
+    except grpc.RpcError as e:
+        # pylint: disable=E1101
+        if e.code() != grpc.StatusCode.DEADLINE_EXCEEDED:
+            raise e
+    finally:
+        if res is not None:
+            latest_timestamp = cast(float, res.latest_timestamp)
 
-    for res in stub.StreamLogs(req):
-        print(res.log_output)
-        if time.time() - start_time > duration:
-            break
+    return max(latest_timestamp, after_timestamp)
 
 
 def print_logs(run_id: int, channel: grpc.Channel, timeout: int) -> None:
@@ -181,11 +210,11 @@ def log(
         )
         raise typer.Exit(code=1)
 
-    _log_with_superexec(federation_config, run_id, stream)
+    _log_with_exec_api(federation_config, run_id, stream)
 
 
 # pylint: disable-next=too-many-branches
-def _log_with_superexec(
+def _log_with_exec_api(
     federation_config: dict[str, str],
     run_id: int,
     stream: bool,

{flwr_nightly-1.13.0.dev20241029 → flwr_nightly-1.13.0.dev20241030}/src/py/flwr/common/constant.py

@@ -87,6 +87,12 @@ MESSAGE_TTL_TOLERANCE = 1e-1
 ISOLATION_MODE_SUBPROCESS = "subprocess"
 ISOLATION_MODE_PROCESS = "process"
 
+# Log streaming configurations
+CONN_REFRESH_PERIOD = 60  # Stream connection refresh period
+CONN_RECONNECT_INTERVAL = 0.5  # Reconnect interval between two stream connections
+LOG_STREAM_INTERVAL = 0.5  # Log stream interval for `ExecServicer.StreamLogs`
+LOG_UPLOAD_INTERVAL = 0.2  # Minimum interval between two log uploads
+
 
 class MessageType:
     """Message type."""

{flwr_nightly-1.13.0.dev20241029 → flwr_nightly-1.13.0.dev20241030}/src/py/flwr/common/logger.py

@@ -17,11 +17,21 @@
 
 import logging
 import sys
+import threading
+import time
 from logging import WARN, LogRecord
 from logging.handlers import HTTPHandler
-from queue import Queue
+from queue import Empty, Queue
 from typing import TYPE_CHECKING, Any, Optional, TextIO
 
+import grpc
+
+from flwr.proto.driver_pb2_grpc import DriverStub  # pylint: disable=E0611
+from flwr.proto.log_pb2 import PushLogsRequest  # pylint: disable=E0611
+from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
+
+from .constant import LOG_UPLOAD_INTERVAL
+
 # Create logger
 LOGGER_NAME = "flwr"
 FLOWER_LOGGER = logging.getLogger(LOGGER_NAME)
@@ -263,7 +273,7 @@ def set_logger_propagation(
     return child_logger
 
 
-def mirror_output_to_queue(log_queue: Queue[str]) -> None:
+def mirror_output_to_queue(log_queue: Queue[Optional[str]]) -> None:
     """Mirror stdout and stderr output to the provided queue."""
 
     def get_write_fn(stream: TextIO) -> Any:
@@ -290,3 +300,65 @@ def restore_output() -> None:
     sys.stdout = sys.__stdout__
     sys.stderr = sys.__stderr__
     console_handler.stream = sys.stdout
+
+
+def _log_uploader(
+    log_queue: Queue[Optional[str]], node_id: int, run_id: int, stub: DriverStub
+) -> None:
+    """Upload logs to the SuperLink."""
+    exit_flag = False
+    node = Node(node_id=node_id, anonymous=False)
+    msgs: list[str] = []
+    while True:
+        # Fetch all messages from the queue
+        try:
+            while True:
+                msg = log_queue.get_nowait()
+                # Quit the loops if the returned message is `None`
+                # This is a signal that the run has finished
+                if msg is None:
+                    exit_flag = True
+                    break
+                msgs.append(msg)
+        except Empty:
+            pass
+
+        # Upload if any logs
+        if msgs:
+            req = PushLogsRequest(
+                node=node,
+                run_id=run_id,
+                logs=msgs,
+            )
+            try:
+                stub.PushLogs(req)
+                msgs.clear()
+            except grpc.RpcError as e:
+                # Ignore minor network errors
+                # pylint: disable-next=no-member
+                if e.code() != grpc.StatusCode.UNAVAILABLE:
+                    raise e
+
+        if exit_flag:
+            break
+
+        time.sleep(LOG_UPLOAD_INTERVAL)
+
+
+def start_log_uploader(
+    log_queue: Queue[Optional[str]], node_id: int, run_id: int, stub: DriverStub
+) -> threading.Thread:
+    """Start the log uploader thread and return it."""
+    thread = threading.Thread(
+        target=_log_uploader, args=(log_queue, node_id, run_id, stub)
+    )
+    thread.start()
+    return thread
+
+
+def stop_log_uploader(
+    log_queue: Queue[Optional[str]], log_uploader: threading.Thread
+) -> None:
+    """Stop the log uploader thread."""
+    log_queue.put(None)
+    log_uploader.join()

{flwr_nightly-1.13.0.dev20241029 → flwr_nightly-1.13.0.dev20241030}/src/py/flwr/proto/driver_pb2.py

@@ -20,7 +20,7 @@ from flwr.proto import run_pb2 as flwr_dot_proto_dot_run__pb2
 from flwr.proto import fab_pb2 as flwr_dot_proto_dot_fab__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17\x66lwr/proto/driver.proto\x12\nflwr.proto\x1a\x14\x66lwr/proto/log.proto\x1a\x15\x66lwr/proto/node.proto\x1a\x18\x66lwr/proto/message.proto\x1a\x15\x66lwr/proto/task.proto\x1a\x14\x66lwr/proto/run.proto\x1a\x14\x66lwr/proto/fab.proto\"!\n\x0fGetNodesRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\x04\"3\n\x10GetNodesResponse\x12\x1f\n\x05nodes\x18\x01 \x03(\x0b\x32\x10.flwr.proto.Node\"@\n\x12PushTaskInsRequest\x12*\n\rtask_ins_list\x18\x01 \x03(\x0b\x32\x13.flwr.proto.TaskIns\"\'\n\x13PushTaskInsResponse\x12\x10\n\x08task_ids\x18\x02 \x03(\t\"F\n\x12PullTaskResRequest\x12\x1e\n\x04node\x18\x01 \x01(\x0b\x32\x10.flwr.proto.Node\x12\x10\n\x08task_ids\x18\x02 \x03(\t\"A\n\x13PullTaskResResponse\x12*\n\rtask_res_list\x18\x01 \x03(\x0b\x32\x13.flwr.proto.TaskRes\",\n\x1aPullServerAppInputsRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\x04\"\x7f\n\x1bPullServerAppInputsResponse\x12$\n\x07\x63ontext\x18\x01 \x01(\x0b\x32\x13.flwr.proto.Context\x12\x1c\n\x03run\x18\x02 \x01(\x0b\x32\x0f.flwr.proto.Run\x12\x1c\n\x03\x66\x61\x62\x18\x03 \x01(\x0b\x32\x0f.flwr.proto.Fab\"S\n\x1bPushServerAppOutputsRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\x04\x12$\n\x07\x63ontext\x18\x02 \x01(\x0b\x32\x13.flwr.proto.Context\"\x1e\n\x1cPushServerAppOutputsResponse2\xc5\x06\n\x06\x44river\x12J\n\tCreateRun\x12\x1c.flwr.proto.CreateRunRequest\x1a\x1d.flwr.proto.CreateRunResponse\"\x00\x12G\n\x08GetNodes\x12\x1b.flwr.proto.GetNodesRequest\x1a\x1c.flwr.proto.GetNodesResponse\"\x00\x12P\n\x0bPushTaskIns\x12\x1e.flwr.proto.PushTaskInsRequest\x1a\x1f.flwr.proto.PushTaskInsResponse\"\x00\x12P\n\x0bPullTaskRes\x12\x1e.flwr.proto.PullTaskResRequest\x1a\x1f.flwr.proto.PullTaskResResponse\"\x00\x12\x41\n\x06GetRun\x12\x19.flwr.proto.GetRunRequest\x1a\x1a.flwr.proto.GetRunResponse\"\x00\x12\x41\n\x06GetFab\x12\x19.flwr.proto.GetFabRequest\x1a\x1a.flwr.proto.GetFabResponse\"\x00\x12h\n\x13PullServerAppInputs\x12&.flwr.proto.PullServerAppInputsRequest\x1a\'.flwr.proto.PullServerAppInputsResponse\"\x00\x12k\n\x14PushServerAppOutputs\x12\'.flwr.proto.PushServerAppOutputsRequest\x1a(.flwr.proto.PushServerAppOutputsResponse\"\x00\x12\\\n\x0fUpdateRunStatus\x12\".flwr.proto.UpdateRunStatusRequest\x1a#.flwr.proto.UpdateRunStatusResponse\"\x00\x12G\n\x08PushLogs\x12\x1b.flwr.proto.PushLogsRequest\x1a\x1c.flwr.proto.PushLogsResponse\"\x00\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17\x66lwr/proto/driver.proto\x12\nflwr.proto\x1a\x14\x66lwr/proto/log.proto\x1a\x15\x66lwr/proto/node.proto\x1a\x18\x66lwr/proto/message.proto\x1a\x15\x66lwr/proto/task.proto\x1a\x14\x66lwr/proto/run.proto\x1a\x14\x66lwr/proto/fab.proto\"!\n\x0fGetNodesRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\x04\"3\n\x10GetNodesResponse\x12\x1f\n\x05nodes\x18\x01 \x03(\x0b\x32\x10.flwr.proto.Node\"@\n\x12PushTaskInsRequest\x12*\n\rtask_ins_list\x18\x01 \x03(\x0b\x32\x13.flwr.proto.TaskIns\"\'\n\x13PushTaskInsResponse\x12\x10\n\x08task_ids\x18\x02 \x03(\t\"F\n\x12PullTaskResRequest\x12\x1e\n\x04node\x18\x01 \x01(\x0b\x32\x10.flwr.proto.Node\x12\x10\n\x08task_ids\x18\x02 \x03(\t\"A\n\x13PullTaskResResponse\x12*\n\rtask_res_list\x18\x01 \x03(\x0b\x32\x13.flwr.proto.TaskRes\"\x1c\n\x1aPullServerAppInputsRequest\"\x7f\n\x1bPullServerAppInputsResponse\x12$\n\x07\x63ontext\x18\x01 \x01(\x0b\x32\x13.flwr.proto.Context\x12\x1c\n\x03run\x18\x02 \x01(\x0b\x32\x0f.flwr.proto.Run\x12\x1c\n\x03\x66\x61\x62\x18\x03 \x01(\x0b\x32\x0f.flwr.proto.Fab\"S\n\x1bPushServerAppOutputsRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\x04\x12$\n\x07\x63ontext\x18\x02 \x01(\x0b\x32\x13.flwr.proto.Context\"\x1e\n\x1cPushServerAppOutputsResponse2\xc5\x06\n\x06\x44river\x12J\n\tCreateRun\x12\x1c.flwr.proto.CreateRunRequest\x1a\x1d.flwr.proto.CreateRunResponse\"\x00\x12G\n\x08GetNodes\x12\x1b.flwr.proto.GetNodesRequest\x1a\x1c.flwr.proto.GetNodesResponse\"\x00\x12P\n\x0bPushTaskIns\x12\x1e.flwr.proto.PushTaskInsRequest\x1a\x1f.flwr.proto.PushTaskInsResponse\"\x00\x12P\n\x0bPullTaskRes\x12\x1e.flwr.proto.PullTaskResRequest\x1a\x1f.flwr.proto.PullTaskResResponse\"\x00\x12\x41\n\x06GetRun\x12\x19.flwr.proto.GetRunRequest\x1a\x1a.flwr.proto.GetRunResponse\"\x00\x12\x41\n\x06GetFab\x12\x19.flwr.proto.GetFabRequest\x1a\x1a.flwr.proto.GetFabResponse\"\x00\x12h\n\x13PullServerAppInputs\x12&.flwr.proto.PullServerAppInputsRequest\x1a\'.flwr.proto.PullServerAppInputsResponse\"\x00\x12k\n\x14PushServerAppOutputs\x12\'.flwr.proto.PushServerAppOutputsRequest\x1a(.flwr.proto.PushServerAppOutputsResponse\"\x00\x12\\\n\x0fUpdateRunStatus\x12\".flwr.proto.UpdateRunStatusRequest\x1a#.flwr.proto.UpdateRunStatusResponse\"\x00\x12G\n\x08PushLogs\x12\x1b.flwr.proto.PushLogsRequest\x1a\x1c.flwr.proto.PushLogsResponse\"\x00\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -40,13 +40,13 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['_PULLTASKRESRESPONSE']._serialized_start=444
   _globals['_PULLTASKRESRESPONSE']._serialized_end=509
   _globals['_PULLSERVERAPPINPUTSREQUEST']._serialized_start=511
-  _globals['_PULLSERVERAPPINPUTSREQUEST']._serialized_end=555
-  _globals['_PULLSERVERAPPINPUTSRESPONSE']._serialized_start=557
-  _globals['_PULLSERVERAPPINPUTSRESPONSE']._serialized_end=684
-  _globals['_PUSHSERVERAPPOUTPUTSREQUEST']._serialized_start=686
-  _globals['_PUSHSERVERAPPOUTPUTSREQUEST']._serialized_end=769
-  _globals['_PUSHSERVERAPPOUTPUTSRESPONSE']._serialized_start=771
-  _globals['_PUSHSERVERAPPOUTPUTSRESPONSE']._serialized_end=801
-  _globals['_DRIVER']._serialized_start=804
-  _globals['_DRIVER']._serialized_end=1641
+  _globals['_PULLSERVERAPPINPUTSREQUEST']._serialized_end=539
+  _globals['_PULLSERVERAPPINPUTSRESPONSE']._serialized_start=541
+  _globals['_PULLSERVERAPPINPUTSRESPONSE']._serialized_end=668
+  _globals['_PUSHSERVERAPPOUTPUTSREQUEST']._serialized_start=670
+  _globals['_PUSHSERVERAPPOUTPUTSREQUEST']._serialized_end=753
+  _globals['_PUSHSERVERAPPOUTPUTSRESPONSE']._serialized_start=755
+  _globals['_PUSHSERVERAPPOUTPUTSRESPONSE']._serialized_end=785
+  _globals['_DRIVER']._serialized_start=788
+  _globals['_DRIVER']._serialized_end=1625
 # @@protoc_insertion_point(module_scope)

{flwr_nightly-1.13.0.dev20241029 → flwr_nightly-1.13.0.dev20241030}/src/py/flwr/proto/driver_pb2.pyi

@@ -98,13 +98,8 @@ global___PullTaskResResponse = PullTaskResResponse
 class PullServerAppInputsRequest(google.protobuf.message.Message):
     """PullServerAppInputs messages"""
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
-    RUN_ID_FIELD_NUMBER: builtins.int
-    run_id: builtins.int
     def __init__(self,
-        *,
-        run_id: builtins.int = ...,
         ) -> None: ...
-    def ClearField(self, field_name: typing_extensions.Literal["run_id",b"run_id"]) -> None: ...
 global___PullServerAppInputsRequest = PullServerAppInputsRequest
 
 class PullServerAppInputsResponse(google.protobuf.message.Message):

{flwr_nightly-1.13.0.dev20241029 → flwr_nightly-1.13.0.dev20241030}/src/py/flwr/proto/exec_pb2.py

@@ -16,7 +16,7 @@ from flwr.proto import fab_pb2 as flwr_dot_proto_dot_fab__pb2
 from flwr.proto import transport_pb2 as flwr_dot_proto_dot_transport__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x15\x66lwr/proto/exec.proto\x12\nflwr.proto\x1a\x14\x66lwr/proto/fab.proto\x1a\x1a\x66lwr/proto/transport.proto\"\xdf\x02\n\x0fStartRunRequest\x12\x1c\n\x03\x66\x61\x62\x18\x01 \x01(\x0b\x32\x0f.flwr.proto.Fab\x12H\n\x0foverride_config\x18\x02 \x03(\x0b\x32/.flwr.proto.StartRunRequest.OverrideConfigEntry\x12L\n\x11\x66\x65\x64\x65ration_config\x18\x03 \x03(\x0b\x32\x31.flwr.proto.StartRunRequest.FederationConfigEntry\x1aI\n\x13OverrideConfigEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12!\n\x05value\x18\x02 \x01(\x0b\x32\x12.flwr.proto.Scalar:\x02\x38\x01\x1aK\n\x15\x46\x65\x64\x65rationConfigEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12!\n\x05value\x18\x02 \x01(\x0b\x32\x12.flwr.proto.Scalar:\x02\x38\x01\"\"\n\x10StartRunResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\x04\"#\n\x11StreamLogsRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\x04\"(\n\x12StreamLogsResponse\x12\x12\n\nlog_output\x18\x01 \x01(\t2\xa0\x01\n\x04\x45xec\x12G\n\x08StartRun\x12\x1b.flwr.proto.StartRunRequest\x1a\x1c.flwr.proto.StartRunResponse\"\x00\x12O\n\nStreamLogs\x12\x1d.flwr.proto.StreamLogsRequest\x1a\x1e.flwr.proto.StreamLogsResponse\"\x00\x30\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x15\x66lwr/proto/exec.proto\x12\nflwr.proto\x1a\x14\x66lwr/proto/fab.proto\x1a\x1a\x66lwr/proto/transport.proto\"\xdf\x02\n\x0fStartRunRequest\x12\x1c\n\x03\x66\x61\x62\x18\x01 \x01(\x0b\x32\x0f.flwr.proto.Fab\x12H\n\x0foverride_config\x18\x02 \x03(\x0b\x32/.flwr.proto.StartRunRequest.OverrideConfigEntry\x12L\n\x11\x66\x65\x64\x65ration_config\x18\x03 \x03(\x0b\x32\x31.flwr.proto.StartRunRequest.FederationConfigEntry\x1aI\n\x13OverrideConfigEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12!\n\x05value\x18\x02 \x01(\x0b\x32\x12.flwr.proto.Scalar:\x02\x38\x01\x1aK\n\x15\x46\x65\x64\x65rationConfigEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12!\n\x05value\x18\x02 \x01(\x0b\x32\x12.flwr.proto.Scalar:\x02\x38\x01\"\"\n\x10StartRunResponse\x12\x0e\n\x06run_id\x18\x01 \x01(\x04\"<\n\x11StreamLogsRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\x04\x12\x17\n\x0f\x61\x66ter_timestamp\x18\x02 \x01(\x01\"B\n\x12StreamLogsResponse\x12\x12\n\nlog_output\x18\x01 \x01(\t\x12\x18\n\x10latest_timestamp\x18\x02 \x01(\x01\x32\xa0\x01\n\x04\x45xec\x12G\n\x08StartRun\x12\x1b.flwr.proto.StartRunRequest\x1a\x1c.flwr.proto.StartRunResponse\"\x00\x12O\n\nStreamLogs\x12\x1d.flwr.proto.StreamLogsRequest\x1a\x1e.flwr.proto.StreamLogsResponse\"\x00\x30\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -36,9 +36,9 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['_STARTRUNRESPONSE']._serialized_start=441
   _globals['_STARTRUNRESPONSE']._serialized_end=475
   _globals['_STREAMLOGSREQUEST']._serialized_start=477
-  _globals['_STREAMLOGSREQUEST']._serialized_end=512
-  _globals['_STREAMLOGSRESPONSE']._serialized_start=514
-  _globals['_STREAMLOGSRESPONSE']._serialized_end=554
-  _globals['_EXEC']._serialized_start=557
-  _globals['_EXEC']._serialized_end=717
+  _globals['_STREAMLOGSREQUEST']._serialized_end=537
+  _globals['_STREAMLOGSRESPONSE']._serialized_start=539
+  _globals['_STREAMLOGSRESPONSE']._serialized_end=605
+  _globals['_EXEC']._serialized_start=608
+  _globals['_EXEC']._serialized_end=768
 # @@protoc_insertion_point(module_scope)

{flwr_nightly-1.13.0.dev20241029 → flwr_nightly-1.13.0.dev20241030}/src/py/flwr/proto/exec_pb2.pyi

@@ -78,21 +78,27 @@ global___StartRunResponse = StartRunResponse
 class StreamLogsRequest(google.protobuf.message.Message):
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
     RUN_ID_FIELD_NUMBER: builtins.int
+    AFTER_TIMESTAMP_FIELD_NUMBER: builtins.int
     run_id: builtins.int
+    after_timestamp: builtins.float
     def __init__(self,
         *,
         run_id: builtins.int = ...,
+        after_timestamp: builtins.float = ...,
         ) -> None: ...
-    def ClearField(self, field_name: typing_extensions.Literal["run_id",b"run_id"]) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["after_timestamp",b"after_timestamp","run_id",b"run_id"]) -> None: ...
 global___StreamLogsRequest = StreamLogsRequest
 
 class StreamLogsResponse(google.protobuf.message.Message):
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
     LOG_OUTPUT_FIELD_NUMBER: builtins.int
+    LATEST_TIMESTAMP_FIELD_NUMBER: builtins.int
     log_output: typing.Text
+    latest_timestamp: builtins.float
     def __init__(self,
         *,
         log_output: typing.Text = ...,
+        latest_timestamp: builtins.float = ...,
         ) -> None: ...
-    def ClearField(self, field_name: typing_extensions.Literal["log_output",b"log_output"]) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["latest_timestamp",b"latest_timestamp","log_output",b"log_output"]) -> None: ...
 global___StreamLogsResponse = StreamLogsResponse

{flwr_nightly-1.13.0.dev20241029 → flwr_nightly-1.13.0.dev20241030}/src/py/flwr/server/app.py

@@ -382,9 +382,7 @@ def _flwr_serverapp_scheduler(
 
             log(
                 INFO,
-                "Launching `flwr-serverapp` subprocess with run-id %d. "
-                "Connects to SuperLink on %s",
-                pending_run_id,
+                "Launching `flwr-serverapp` subprocess. Connects to SuperLink on %s",
                 driver_api_address,
             )
             # Start ServerApp subprocess
@@ -392,8 +390,6 @@ def _flwr_serverapp_scheduler(
                 "flwr-serverapp",
                 "--superlink",
                 driver_api_address,
-                "--run-id",
-                str(pending_run_id),
             ]
             if ssl_ca_certfile:
                 command.append("--root-certificates")

{flwr_nightly-1.13.0.dev20241029 → flwr_nightly-1.13.0.dev20241030}/src/py/flwr/server/serverapp/app.py

@@ -19,6 +19,7 @@ import sys
 from logging import DEBUG, ERROR, INFO, WARN
 from os.path import isfile
 from pathlib import Path
+from queue import Queue
 from time import sleep
 from typing import Optional
 
@@ -31,7 +32,13 @@ from flwr.common.config import (
     get_project_dir,
 )
 from flwr.common.constant import Status, SubStatus
-from flwr.common.logger import log
+from flwr.common.logger import (
+    log,
+    mirror_output_to_queue,
+    restore_output,
+    start_log_uploader,
+    stop_log_uploader,
+)
 from flwr.common.serde import (
     context_from_proto,
     context_to_proto,
@@ -52,6 +59,10 @@ from flwr.server.run_serverapp import run as run_
 
 def flwr_serverapp() -> None:
     """Run process-isolated Flower ServerApp."""
+    # Capture stdout/stderr
+    log_queue: Queue[Optional[str]] = Queue()
+    mirror_output_to_queue(log_queue)
+
     log(INFO, "Starting Flower ServerApp")
 
     parser = argparse.ArgumentParser(
@@ -63,11 +74,10 @@ def flwr_serverapp() -> None:
         help="Address of SuperLink's DriverAPI",
     )
     parser.add_argument(
-        "--run-id",
-        type=int,
-        required=False,
-        help="Id of the Run this process should start. If not supplied, this "
-        "function will request a pending run to the LinkState.",
+        "--run-once",
+        action="store_true",
+        help="When set, this process will start a single ServerApp "
+        "for a pending Run. If no pending run the process will exit. ",
     )
     parser.add_argument(
         "--flwr-dir",
@@ -100,18 +110,20 @@ def flwr_serverapp() -> None:
 
     log(
         DEBUG,
-        "Staring isolated `ServerApp` connected to SuperLink DriverAPI at %s "
-        "for run-id %s",
+        "Staring isolated `ServerApp` connected to SuperLink DriverAPI at %s",
         args.superlink,
-        args.run_id,
     )
     run_serverapp(
         superlink=args.superlink,
-        run_id=args.run_id,
+        log_queue=log_queue,
+        run_once=args.run_once,
         flwr_dir_=args.flwr_dir,
         certificates=certificates,
     )
 
+    # Restore stdout/stderr
+    restore_output()
+
 
 def _try_obtain_certificates(
     args: argparse.Namespace,
@@ -148,7 +160,8 @@ def _try_obtain_certificates(
 
 def run_serverapp(  # pylint: disable=R0914, disable=W0212
     superlink: str,
-    run_id: Optional[int] = None,
+    log_queue: Queue[Optional[str]],
+    run_once: bool,
     flwr_dir_: Optional[str] = None,
     certificates: Optional[bytes] = None,
 ) -> None:
@@ -160,18 +173,13 @@ def run_serverapp(  # pylint: disable=R0914, disable=W0212
 
     # Resolve directory where FABs are installed
     flwr_dir = get_flwr_dir(flwr_dir_)
-
-    only_once = run_id is not None
+    log_uploader = None
 
     while True:
 
         try:
             # Pull ServerAppInputs from LinkState
-            req = (
-                PullServerAppInputsRequest(run_id=run_id)
-                if run_id
-                else PullServerAppInputsRequest()
-            )
+            req = PullServerAppInputsRequest()
             res: PullServerAppInputsResponse = driver._stub.PullServerAppInputs(req)
             if not res.HasField("run"):
                 sleep(3)
@@ -184,6 +192,14 @@ def run_serverapp(  # pylint: disable=R0914, disable=W0212
 
             driver.init_run(run.run_id)
 
+            # Start log uploader for this run
+            log_uploader = start_log_uploader(
+                log_queue=log_queue,
+                node_id=0,
+                run_id=run.run_id,
+                stub=driver._stub,
+            )
+
             log(DEBUG, "ServerApp process starts FAB installation.")
             install_from_fab(fab.content, flwr_dir=flwr_dir, skip_prompt=True)
 
@@ -245,9 +261,11 @@ def run_serverapp(  # pylint: disable=R0914, disable=W0212
                     )
                 )
 
+            # Stop log uploader for this run
+            if log_uploader:
+                stop_log_uploader(log_queue, log_uploader)
+                log_uploader = None
+
         # Stop the loop if `flwr-serverapp` is expected to process a single run
-        if only_once:
+        if run_once:
             break
-
-        # Reset the run_id
-        run_id = None

{flwr_nightly-1.13.0.dev20241029 → flwr_nightly-1.13.0.dev20241030}/src/py/flwr/server/superlink/driver/driver_servicer.py

@@ -219,11 +219,8 @@ class DriverServicer(driver_pb2_grpc.DriverServicer):
 
         # Lock access to LinkState, preventing obtaining the same pending run_id
         with self.lock:
-            # If run_id is provided, use it, otherwise use the pending run_id
-            if _has_field(request, "run_id"):
-                run_id: Optional[int] = request.run_id
-            else:
-                run_id = state.get_pending_run_id()
+            # Attempt getting the run_id of a pending run
+            run_id = state.get_pending_run_id()
             # If there's no pending run, return an empty response
             if run_id is None:
                 return PullServerAppInputsResponse()
@@ -235,14 +232,12 @@ class DriverServicer(driver_pb2_grpc.DriverServicer):
             if run and run.fab_hash:
                 if result := ffs.get(run.fab_hash):
                     fab = Fab(run.fab_hash, result[0])
-            if run and fab:
+            if run and fab and serverapp_ctxt:
                 # Update run status to STARTING
                 if state.update_run_status(run_id, RunStatus(Status.STARTING, "", "")):
                     log(INFO, "Starting run %d", run_id)
                     return PullServerAppInputsResponse(
-                        context=(
-                            context_to_proto(serverapp_ctxt) if serverapp_ctxt else None
-                        ),
+                        context=context_to_proto(serverapp_ctxt),
                         run=run_to_proto(run),
                         fab=fab_to_proto(fab),
                     )
@@ -278,7 +273,12 @@ class DriverServicer(driver_pb2_grpc.DriverServicer):
     ) -> PushLogsResponse:
         """Push logs."""
         log(DEBUG, "DriverServicer.PushLogs")
-        raise NotImplementedError()
+        state = self.state_factory.state()
+
+        # Add logs to LinkState
+        merged_logs = "".join(request.logs)
+        state.add_serverapp_log(request.run_id, merged_logs)
+        return PushLogsResponse()
 
 
 def _raise_if(validation_error: bool, detail: str) -> None:

{flwr_nightly-1.13.0.dev20241029 → flwr_nightly-1.13.0.dev20241030}/src/py/flwr/superexec/deployment.py

@@ -21,9 +21,10 @@ from typing import Optional
 
 from typing_extensions import override
 
-from flwr.common.constant import DRIVER_API_DEFAULT_ADDRESS
+from flwr.common import Context, RecordSet
+from flwr.common.constant import DRIVER_API_DEFAULT_ADDRESS, Status, SubStatus
 from flwr.common.logger import log
-from flwr.common.typing import Fab, UserConfig
+from flwr.common.typing import Fab, RunStatus, UserConfig
 from flwr.server.superlink.ffs import Ffs
 from flwr.server.superlink.ffs.ffs_factory import FfsFactory
 from flwr.server.superlink.linkstate import LinkState, LinkStateFactory
@@ -135,6 +136,14 @@ class DeploymentEngine(Executor):
         run_id = self.linkstate.create_run(None, None, fab_hash, override_config)
         return run_id
 
+    def _create_context(self, run_id: int) -> None:
+        """Register a Context for a Run."""
+        # Create an empty context for the Run
+        context = Context(node_id=0, node_config={}, state=RecordSet(), run_config={})
+
+        # Register the context at the LinkState
+        self.linkstate.set_serverapp_context(run_id=run_id, context=context)
+
     @override
     def start_run(
         self,
@@ -143,18 +152,25 @@ class DeploymentEngine(Executor):
         federation_config: UserConfig,
     ) -> Optional[int]:
         """Start run using the Flower Deployment Engine."""
+        run_id = None
         try:
 
             # Call SuperLink to create run
-            run_id: int = self._create_run(
+            run_id = self._create_run(
                 Fab(hashlib.sha256(fab_file).hexdigest(), fab_file), override_config
             )
+
+            # Register context for the Run
+            self._create_context(run_id=run_id)
             log(INFO, "Created run %s", str(run_id))
 
             return run_id
         # pylint: disable-next=broad-except
         except Exception as e:
             log(ERROR, "Could not start run: %s", str(e))
+            if run_id:
+                run_status = RunStatus(Status.FINISHED, SubStatus.FAILED, str(e))
+                self.linkstate.update_run_status(run_id, new_status=run_status)
             return None