flwr-nightly 1.8.0.dev20240401__tar.gz → 1.9.0.dev20240403__tar.gz

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.8.0.dev20240401
+Version: 1.9.0.dev20240403
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "flwr-nightly"
-version = "1.8.0-dev20240401"
+version = "1.9.0-dev20240403"
 description = "Flower: A Friendly Federated Learning Framework"
 license = "Apache-2.0"
 authors = ["The Flower Authors <hello@flower.ai>"]

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/src/py/flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl

@@ -15,7 +15,7 @@ readme = "README.md"
 [tool.poetry.dependencies]
 python = "^3.9"
 # Mandatory dependencies
-flwr-nightly = { version = "1.8.0.dev20240313", extras = ["simulation"] }
+flwr = { version = "^1.8.0", extras = ["simulation"] }
 flwr-datasets = { version = "0.0.2", extras = ["vision"] }
 torch = "2.2.1"
 torchvision = "0.17.1"

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/src/py/flwr/cli/new/templates/app/requirements.pytorch.txt.tpl

@@ -1,4 +1,4 @@
-flwr-nightly[simulation]==1.8.0.dev20240313
+flwr[simulation]>=1.8.0
 flwr-datasets[vision]==0.0.2
 torch==2.2.1
 torchvision==0.17.1

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/src/py/flwr/client/app.py

@@ -34,6 +34,7 @@ from flwr.common.constant import (
     TRANSPORT_TYPE_GRPC_RERE,
     TRANSPORT_TYPE_REST,
     TRANSPORT_TYPES,
+    ErrorCode,
 )
 from flwr.common.exit_handlers import register_exit_handlers
 from flwr.common.logger import log, warn_deprecated_feature
@@ -483,7 +484,7 @@ def _start_client_internal(
                 # Create an error reply message that will never be used to prevent
                 # the used-before-assignment linting error
                 reply_message = message.create_error_reply(
-                    error=Error(code=0, reason="Unknown")
+                    error=Error(code=ErrorCode.UNKNOWN, reason="Unknown")
                 )
 
                 # Handle app loading and task message
@@ -491,27 +492,41 @@ def _start_client_internal(
                     # Load ClientApp instance
                     client_app: ClientApp = load_client_app_fn()
 
+                    # Execute ClientApp
                     reply_message = client_app(message=message, context=context)
-                    # Update node state
-                    node_state.update_context(
-                        run_id=message.metadata.run_id,
-                        context=context,
-                    )
                 except Exception as ex:  # pylint: disable=broad-exception-caught
-                    log(ERROR, "ClientApp raised an exception", exc_info=ex)
 
                     # Legacy grpc-bidi
                     if transport in ["grpc-bidi", None]:
+                        log(ERROR, "Client raised an exception.", exc_info=ex)
                         # Raise exception, crash process
                         raise ex
 
                     # Don't update/change NodeState
 
-                    # Create error message
+                    e_code = ErrorCode.CLIENT_APP_RAISED_EXCEPTION
                     # Reason example: "<class 'ZeroDivisionError'>:<'division by zero'>"
                     reason = str(type(ex)) + ":<'" + str(ex) + "'>"
+                    exc_entity = "ClientApp"
+                    if isinstance(ex, LoadClientAppError):
+                        reason = (
+                            "An exception was raised when attempting to load "
+                            "`ClientApp`"
+                        )
+                        e_code = ErrorCode.LOAD_CLIENT_APP_EXCEPTION
+                        exc_entity = "SuperNode"
+
+                    log(ERROR, "%s raised an exception", exc_entity, exc_info=ex)
+
+                    # Create error message
                     reply_message = message.create_error_reply(
-                        error=Error(code=0, reason=reason)
+                        error=Error(code=e_code, reason=reason)
+                    )
+                else:
+                    # No exception, update node state
+                    node_state.update_context(
+                        run_id=message.metadata.run_id,
+                        context=context,
                     )
 
                 # Send

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/src/py/flwr/client/client_app.py

@@ -28,6 +28,15 @@ from flwr.common.logger import warn_preview_feature
 from .typing import ClientAppCallable
 
 
+class ClientAppException(Exception):
+    """Exception raised when an exception is raised while executing a ClientApp."""
+
+    def __init__(self, message: str):
+        ex_name = self.__class__.__name__
+        self.message = f"\nException {ex_name} occurred. Message: " + message
+        super().__init__(self.message)
+
+
 class ClientApp:
     """Flower ClientApp.
 

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/src/py/flwr/client/grpc_rere_client/connection.py

@@ -151,7 +151,7 @@ def grpc_request_response(  # pylint: disable=R0914, R0915
     def create_node() -> None:
         """Set create_node."""
         # Call FleetAPI
-        create_node_request = CreateNodeRequest()
+        create_node_request = CreateNodeRequest(ping_interval=PING_DEFAULT_INTERVAL)
         create_node_response = retry_invoker.invoke(
             stub.CreateNode,
             request=create_node_request,

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/src/py/flwr/client/rest_client/connection.py

@@ -201,7 +201,7 @@ def http_request_response(  # pylint: disable=R0914, R0915
 
     def create_node() -> None:
         """Set create_node."""
-        create_node_req_proto = CreateNodeRequest()
+        create_node_req_proto = CreateNodeRequest(ping_interval=PING_DEFAULT_INTERVAL)
         create_node_req_bytes: bytes = create_node_req_proto.SerializeToString()
 
         res = retry_invoker.invoke(

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/src/py/flwr/common/constant.py

@@ -41,6 +41,7 @@ PING_DEFAULT_INTERVAL = 30
 PING_CALL_TIMEOUT = 5
 PING_BASE_MULTIPLIER = 0.8
 PING_RANDOM_RANGE = (-0.1, 0.1)
+PING_MAX_INTERVAL = 1e300
 
 
 class MessageType:
@@ -74,3 +75,16 @@ class SType:
     def __new__(cls) -> SType:
         """Prevent instantiation."""
         raise TypeError(f"{cls.__name__} cannot be instantiated.")
+
+
+class ErrorCode:
+    """Error codes for Message's Error."""
+
+    UNKNOWN = 0
+    LOAD_CLIENT_APP_EXCEPTION = 1
+    CLIENT_APP_RAISED_EXCEPTION = 2
+    NODE_UNAVAILABLE = 3
+
+    def __new__(cls) -> ErrorCode:
+        """Prevent instantiation."""
+        raise TypeError(f"{cls.__name__} cannot be instantiated.")

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/src/py/flwr/common/message.py

@@ -17,6 +17,7 @@
 from __future__ import annotations
 
 import time
+import warnings
 from dataclasses import dataclass
 
 from .record import RecordSet
@@ -311,6 +312,13 @@ class Message:
 
             ttl = msg.meta.ttl - (reply.meta.created_at - msg.meta.created_at)
         """
+        if ttl:
+            warnings.warn(
+                "A custom TTL was set, but note that the SuperLink does not enforce "
+                "the TTL yet. The SuperLink will start enforcing the TTL in a future "
+                "version of Flower.",
+                stacklevel=2,
+            )
         # If no TTL passed, use default for message creation (will update after
         # message creation)
         ttl_ = DEFAULT_TTL if ttl is None else ttl
@@ -349,6 +357,13 @@ class Message:
         Message
             A new `Message` instance representing the reply.
         """
+        if ttl:
+            warnings.warn(
+                "A custom TTL was set, but note that the SuperLink does not enforce "
+                "the TTL yet. The SuperLink will start enforcing the TTL in a future "
+                "version of Flower.",
+                stacklevel=2,
+            )
         # If no TTL passed, use default for message creation (will update after
         # message creation)
         ttl_ = DEFAULT_TTL if ttl is None else ttl

flwr_nightly-1.9.0.dev20240403/src/py/flwr/proto/fleet_pb2.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: flwr/proto/fleet.proto
+# Protobuf Python Version: 4.25.0
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from flwr.proto import node_pb2 as flwr_dot_proto_dot_node__pb2
+from flwr.proto import task_pb2 as flwr_dot_proto_dot_task__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16\x66lwr/proto/fleet.proto\x12\nflwr.proto\x1a\x15\x66lwr/proto/node.proto\x1a\x15\x66lwr/proto/task.proto\"*\n\x11\x43reateNodeRequest\x12\x15\n\rping_interval\x18\x01 \x01(\x01\"4\n\x12\x43reateNodeResponse\x12\x1e\n\x04node\x18\x01 \x01(\x0b\x32\x10.flwr.proto.Node\"3\n\x11\x44\x65leteNodeRequest\x12\x1e\n\x04node\x18\x01 \x01(\x0b\x32\x10.flwr.proto.Node\"\x14\n\x12\x44\x65leteNodeResponse\"D\n\x0bPingRequest\x12\x1e\n\x04node\x18\x01 \x01(\x0b\x32\x10.flwr.proto.Node\x12\x15\n\rping_interval\x18\x02 \x01(\x01\"\x1f\n\x0cPingResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\"F\n\x12PullTaskInsRequest\x12\x1e\n\x04node\x18\x01 \x01(\x0b\x32\x10.flwr.proto.Node\x12\x10\n\x08task_ids\x18\x02 \x03(\t\"k\n\x13PullTaskInsResponse\x12(\n\treconnect\x18\x01 \x01(\x0b\x32\x15.flwr.proto.Reconnect\x12*\n\rtask_ins_list\x18\x02 \x03(\x0b\x32\x13.flwr.proto.TaskIns\"@\n\x12PushTaskResRequest\x12*\n\rtask_res_list\x18\x01 \x03(\x0b\x32\x13.flwr.proto.TaskRes\"\xae\x01\n\x13PushTaskResResponse\x12(\n\treconnect\x18\x01 \x01(\x0b\x32\x15.flwr.proto.Reconnect\x12=\n\x07results\x18\x02 \x03(\x0b\x32,.flwr.proto.PushTaskResResponse.ResultsEntry\x1a.\n\x0cResultsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\r:\x02\x38\x01\"\x1e\n\tReconnect\x12\x11\n\treconnect\x18\x01 \x01(\x04\x32\x86\x03\n\x05\x46leet\x12M\n\nCreateNode\x12\x1d.flwr.proto.CreateNodeRequest\x1a\x1e.flwr.proto.CreateNodeResponse\"\x00\x12M\n\nDeleteNode\x12\x1d.flwr.proto.DeleteNodeRequest\x1a\x1e.flwr.proto.DeleteNodeResponse\"\x00\x12;\n\x04Ping\x12\x17.flwr.proto.PingRequest\x1a\x18.flwr.proto.PingResponse\"\x00\x12P\n\x0bPullTaskIns\x12\x1e.flwr.proto.PullTaskInsRequest\x1a\x1f.flwr.proto.PullTaskInsResponse\"\x00\x12P\n\x0bPushTaskRes\x12\x1e.flwr.proto.PushTaskResRequest\x1a\x1f.flwr.proto.PushTaskResResponse\"\x00\x62\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'flwr.proto.fleet_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+  DESCRIPTOR._options = None
+  _globals['_PUSHTASKRESRESPONSE_RESULTSENTRY']._options = None
+  _globals['_PUSHTASKRESRESPONSE_RESULTSENTRY']._serialized_options = b'8\001'
+  _globals['_CREATENODEREQUEST']._serialized_start=84
+  _globals['_CREATENODEREQUEST']._serialized_end=126
+  _globals['_CREATENODERESPONSE']._serialized_start=128
+  _globals['_CREATENODERESPONSE']._serialized_end=180
+  _globals['_DELETENODEREQUEST']._serialized_start=182
+  _globals['_DELETENODEREQUEST']._serialized_end=233
+  _globals['_DELETENODERESPONSE']._serialized_start=235
+  _globals['_DELETENODERESPONSE']._serialized_end=255
+  _globals['_PINGREQUEST']._serialized_start=257
+  _globals['_PINGREQUEST']._serialized_end=325
+  _globals['_PINGRESPONSE']._serialized_start=327
+  _globals['_PINGRESPONSE']._serialized_end=358
+  _globals['_PULLTASKINSREQUEST']._serialized_start=360
+  _globals['_PULLTASKINSREQUEST']._serialized_end=430
+  _globals['_PULLTASKINSRESPONSE']._serialized_start=432
+  _globals['_PULLTASKINSRESPONSE']._serialized_end=539
+  _globals['_PUSHTASKRESREQUEST']._serialized_start=541
+  _globals['_PUSHTASKRESREQUEST']._serialized_end=605
+  _globals['_PUSHTASKRESRESPONSE']._serialized_start=608
+  _globals['_PUSHTASKRESRESPONSE']._serialized_end=782
+  _globals['_PUSHTASKRESRESPONSE_RESULTSENTRY']._serialized_start=736
+  _globals['_PUSHTASKRESRESPONSE_RESULTSENTRY']._serialized_end=782
+  _globals['_RECONNECT']._serialized_start=784
+  _globals['_RECONNECT']._serialized_end=814
+  _globals['_FLEET']._serialized_start=817
+  _globals['_FLEET']._serialized_end=1207
+# @@protoc_insertion_point(module_scope)

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/src/py/flwr/proto/fleet_pb2.pyi

@@ -16,8 +16,13 @@ DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
 class CreateNodeRequest(google.protobuf.message.Message):
     """CreateNode messages"""
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
+    PING_INTERVAL_FIELD_NUMBER: builtins.int
+    ping_interval: builtins.float
     def __init__(self,
+        *,
+        ping_interval: builtins.float = ...,
         ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["ping_interval",b"ping_interval"]) -> None: ...
 global___CreateNodeRequest = CreateNodeRequest
 
 class CreateNodeResponse(google.protobuf.message.Message):

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/src/py/flwr/server/driver/driver.py

@@ -14,8 +14,8 @@
 # ==============================================================================
 """Flower driver service client."""
 
-
 import time
+import warnings
 from typing import Iterable, List, Optional, Tuple
 
 from flwr.common import DEFAULT_TTL, Message, Metadata, RecordSet
@@ -91,7 +91,7 @@ class Driver:
         message_type: str,
         dst_node_id: int,
         group_id: str,
-        ttl: float = DEFAULT_TTL,
+        ttl: Optional[float] = None,
     ) -> Message:
         """Create a new message with specified parameters.
 
@@ -111,10 +111,11 @@ class Driver:
         group_id : str
             The ID of the group to which this message is associated. In some settings,
             this is used as the FL round.
-        ttl : float (default: common.DEFAULT_TTL)
+        ttl : Optional[float] (default: None)
             Time-to-live for the round trip of this message, i.e., the time from sending
             this message to receiving a reply. It specifies in seconds the duration for
-            which the message and its potential reply are considered valid.
+            which the message and its potential reply are considered valid. If unset,
+            the default TTL (i.e., `common.DEFAULT_TTL`) will be used.
 
         Returns
         -------
@@ -122,6 +123,15 @@ class Driver:
             A new `Message` instance with the specified content and metadata.
         """
         _, run_id = self._get_grpc_driver_and_run_id()
+        if ttl:
+            warnings.warn(
+                "A custom TTL was set, but note that the SuperLink does not enforce "
+                "the TTL yet. The SuperLink will start enforcing the TTL in a future "
+                "version of Flower.",
+                stacklevel=2,
+            )
+
+        ttl_ = DEFAULT_TTL if ttl is None else ttl
         metadata = Metadata(
             run_id=run_id,
             message_id="",  # Will be set by the server
@@ -129,7 +139,7 @@ class Driver:
             dst_node_id=dst_node_id,
             reply_to_message="",
             group_id=group_id,
-            ttl=ttl,
+            ttl=ttl_,
             message_type=message_type,
         )
         return Message(metadata=metadata, content=content)

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/src/py/flwr/server/superlink/fleet/message_handler/message_handler.py

@@ -43,7 +43,7 @@ def create_node(
 ) -> CreateNodeResponse:
     """."""
     # Create node
-    node_id = state.create_node()
+    node_id = state.create_node(ping_interval=request.ping_interval)
     return CreateNodeResponse(node=Node(node_id=node_id, anonymous=False))
 
 

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/src/py/flwr/server/superlink/fleet/vce/vce_api.py

@@ -22,8 +22,9 @@ import traceback
 from logging import DEBUG, ERROR, INFO, WARN
 from typing import Callable, Dict, List, Optional
 
-from flwr.client.client_app import ClientApp, LoadClientAppError
+from flwr.client.client_app import ClientApp, ClientAppException, LoadClientAppError
 from flwr.client.node_state import NodeState
+from flwr.common.constant import PING_MAX_INTERVAL, ErrorCode
 from flwr.common.logger import log
 from flwr.common.message import Error
 from flwr.common.object_ref import load_app
@@ -43,7 +44,7 @@ def _register_nodes(
     nodes_mapping: NodeToPartitionMapping = {}
     state = state_factory.state()
     for i in range(num_nodes):
-        node_id = state.create_node()
+        node_id = state.create_node(ping_interval=PING_MAX_INTERVAL)
         nodes_mapping[node_id] = i
     log(INFO, "Registered %i nodes", len(nodes_mapping))
     return nodes_mapping
@@ -93,9 +94,18 @@ async def worker(
         except Exception as ex:  # pylint: disable=broad-exception-caught
             log(ERROR, ex)
             log(ERROR, traceback.format_exc())
+
+            if isinstance(ex, ClientAppException):
+                e_code = ErrorCode.CLIENT_APP_RAISED_EXCEPTION
+            elif isinstance(ex, LoadClientAppError):
+                e_code = ErrorCode.LOAD_CLIENT_APP_EXCEPTION
+            else:
+                e_code = ErrorCode.UNKNOWN
+
             reason = str(type(ex)) + ":<'" + str(ex) + "'>"
-            error = Error(code=0, reason=reason)
-            out_mssg = message.create_error_reply(error=error)
+            out_mssg = message.create_error_reply(
+                error=Error(code=e_code, reason=reason)
+            )
 
         finally:
             if out_mssg:

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/src/py/flwr/server/superlink/state/in_memory_state.py

@@ -27,6 +27,8 @@ from flwr.proto.task_pb2 import TaskIns, TaskRes  # pylint: disable=E0611
 from flwr.server.superlink.state.state import State
 from flwr.server.utils import validate_task_ins_or_res
 
+from .utils import make_node_unavailable_taskres
+
 
 class InMemoryState(State):
     """In-memory State implementation."""
@@ -129,15 +131,32 @@ class InMemoryState(State):
         with self.lock:
             # Find TaskRes that were not delivered yet
             task_res_list: List[TaskRes] = []
+            replied_task_ids: Set[UUID] = set()
             for _, task_res in self.task_res_store.items():
-                if (
-                    UUID(task_res.task.ancestry[0]) in task_ids
-                    and task_res.task.delivered_at == ""
-                ):
+                reply_to = UUID(task_res.task.ancestry[0])
+                if reply_to in task_ids and task_res.task.delivered_at == "":
                     task_res_list.append(task_res)
+                    replied_task_ids.add(reply_to)
                 if limit and len(task_res_list) == limit:
                     break
 
+            # Check if the node is offline
+            for task_id in task_ids - replied_task_ids:
+                if limit and len(task_res_list) == limit:
+                    break
+                task_ins = self.task_ins_store.get(task_id)
+                if task_ins is None:
+                    continue
+                node_id = task_ins.task.consumer.node_id
+                online_until, _ = self.node_ids[node_id]
+                # Generate a TaskRes containing an error reply if the node is offline.
+                if online_until < time.time():
+                    err_taskres = make_node_unavailable_taskres(
+                        ref_taskins=task_ins,
+                    )
+                    self.task_res_store[UUID(err_taskres.task_id)] = err_taskres
+                    task_res_list.append(err_taskres)
+
             # Mark all of them as delivered
             delivered_at = now().isoformat()
             for task_res in task_res_list:
@@ -182,16 +201,14 @@ class InMemoryState(State):
         """
         return len(self.task_res_store)
 
-    def create_node(self) -> int:
+    def create_node(self, ping_interval: float) -> int:
         """Create, store in state, and return `node_id`."""
         # Sample a random int64 as node_id
         node_id: int = int.from_bytes(os.urandom(8), "little", signed=True)
 
         with self.lock:
             if node_id not in self.node_ids:
-                # Default ping interval is 30s
-                # TODO: change 1e9 to 30s  # pylint: disable=W0511
-                self.node_ids[node_id] = (time.time() + 1e9, 1e9)
+                self.node_ids[node_id] = (time.time() + ping_interval, ping_interval)
                 return node_id
         log(ERROR, "Unexpected node registration failure.")
         return 0

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/src/py/flwr/server/superlink/state/sqlite_state.py

@@ -30,6 +30,7 @@ from flwr.proto.task_pb2 import Task, TaskIns, TaskRes  # pylint: disable=E0611
 from flwr.server.utils.validator import validate_task_ins_or_res
 
 from .state import State
+from .utils import make_node_unavailable_taskres
 
 SQL_CREATE_TABLE_NODE = """
 CREATE TABLE IF NOT EXISTS node(
@@ -344,6 +345,7 @@ class SqliteState(State):
 
         return task_id
 
+    # pylint: disable-next=R0914
     def get_task_res(self, task_ids: Set[UUID], limit: Optional[int]) -> List[TaskRes]:
         """Get TaskRes for task_ids.
 
@@ -374,7 +376,7 @@ class SqliteState(State):
             AND delivered_at = ""
         """
 
-        data: Dict[str, Union[str, int]] = {}
+        data: Dict[str, Union[str, float, int]] = {}
 
         if limit is not None:
             query += " LIMIT :limit"
@@ -408,6 +410,54 @@ class SqliteState(State):
             rows = self.query(query, data)
 
         result = [dict_to_task_res(row) for row in rows]
+
+        # 1. Query: Fetch consumer_node_id of remaining task_ids
+        # Assume the ancestry field only contains one element
+        data.clear()
+        replied_task_ids: Set[UUID] = {UUID(str(row["ancestry"])) for row in rows}
+        remaining_task_ids = task_ids - replied_task_ids
+        placeholders = ",".join([f":id_{i}" for i in range(len(remaining_task_ids))])
+        query = f"""
+            SELECT consumer_node_id
+            FROM task_ins
+            WHERE task_id IN ({placeholders});
+        """
+        for index, task_id in enumerate(remaining_task_ids):
+            data[f"id_{index}"] = str(task_id)
+        node_ids = [int(row["consumer_node_id"]) for row in self.query(query, data)]
+
+        # 2. Query: Select offline nodes
+        placeholders = ",".join([f":id_{i}" for i in range(len(node_ids))])
+        query = f"""
+            SELECT node_id
+            FROM node
+            WHERE node_id IN ({placeholders})
+            AND online_until < :time;
+        """
+        data = {f"id_{i}": str(node_id) for i, node_id in enumerate(node_ids)}
+        data["time"] = time.time()
+        offline_node_ids = [int(row["node_id"]) for row in self.query(query, data)]
+
+        # 3. Query: Select TaskIns for offline nodes
+        placeholders = ",".join([f":id_{i}" for i in range(len(offline_node_ids))])
+        query = f"""
+            SELECT *
+            FROM task_ins
+            WHERE consumer_node_id IN ({placeholders});
+        """
+        data = {f"id_{i}": str(node_id) for i, node_id in enumerate(offline_node_ids)}
+        task_ins_rows = self.query(query, data)
+
+        # Make TaskRes containing node unavailabe error
+        for row in task_ins_rows:
+            if limit and len(result) == limit:
+                break
+            task_ins = dict_to_task_ins(row)
+            err_taskres = make_node_unavailable_taskres(
+                ref_taskins=task_ins,
+            )
+            result.append(err_taskres)
+
         return result
 
     def num_task_ins(self) -> int:
@@ -468,7 +518,7 @@ class SqliteState(State):
 
         return None
 
-    def create_node(self) -> int:
+    def create_node(self, ping_interval: float) -> int:
         """Create, store in state, and return `node_id`."""
         # Sample a random int64 as node_id
         node_id: int = int.from_bytes(os.urandom(8), "little", signed=True)
@@ -478,9 +528,7 @@ class SqliteState(State):
         )
 
         try:
-            # Default ping interval is 30s
-            # TODO: change 1e9 to 30s  # pylint: disable=W0511
-            self.query(query, (node_id, time.time() + 1e9, 1e9))
+            self.query(query, (node_id, time.time() + ping_interval, ping_interval))
         except sqlite3.IntegrityError:
             log(ERROR, "Unexpected node registration failure.")
             return 0

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/src/py/flwr/server/superlink/state/state.py

@@ -132,7 +132,7 @@ class State(abc.ABC):
         """Delete all delivered TaskIns/TaskRes pairs."""
 
     @abc.abstractmethod
-    def create_node(self) -> int:
+    def create_node(self, ping_interval: float) -> int:
         """Create, store in state, and return `node_id`."""
 
     @abc.abstractmethod

flwr_nightly-1.9.0.dev20240403/src/py/flwr/server/superlink/state/utils.py

@@ -0,0 +1,56 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utility functions for State."""
+
+
+import time
+from logging import ERROR
+from uuid import uuid4
+
+from flwr.common import log
+from flwr.common.constant import ErrorCode
+from flwr.proto.error_pb2 import Error  # pylint: disable=E0611
+from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
+from flwr.proto.task_pb2 import Task, TaskIns, TaskRes  # pylint: disable=E0611
+
+NODE_UNAVAILABLE_ERROR_REASON = (
+    "Error: Node Unavailable - The destination node is currently unavailable. "
+    "It exceeds the time limit specified in its last ping."
+)
+
+
+def make_node_unavailable_taskres(ref_taskins: TaskIns) -> TaskRes:
+    """Generate a TaskRes with a node unavailable error from a TaskIns."""
+    current_time = time.time()
+    ttl = ref_taskins.task.ttl - (current_time - ref_taskins.task.created_at)
+    if ttl < 0:
+        log(ERROR, "Creating TaskRes for TaskIns that exceeds its TTL.")
+        ttl = 0
+    return TaskRes(
+        task_id=str(uuid4()),
+        group_id=ref_taskins.group_id,
+        run_id=ref_taskins.run_id,
+        task=Task(
+            producer=Node(node_id=ref_taskins.task.consumer.node_id, anonymous=False),
+            consumer=Node(node_id=ref_taskins.task.producer.node_id, anonymous=False),
+            created_at=current_time,
+            ttl=ttl,
+            ancestry=[ref_taskins.task_id],
+            task_type=ref_taskins.task.task_type,
+            error=Error(
+                code=ErrorCode.NODE_UNAVAILABLE, reason=NODE_UNAVAILABLE_ERROR_REASON
+            ),
+        ),
+    )

{flwr_nightly-1.8.0.dev20240401 → flwr_nightly-1.9.0.dev20240403}/src/py/flwr/server/workflow/default_workflows.py

@@ -21,7 +21,7 @@ from logging import INFO
 from typing import Optional, cast
 
 import flwr.common.recordset_compat as compat
-from flwr.common import DEFAULT_TTL, ConfigsRecord, Context, GetParametersIns, log
+from flwr.common import ConfigsRecord, Context, GetParametersIns, log
 from flwr.common.constant import MessageType, MessageTypeLegacy
 
 from ..compat.app_utils import start_update_client_manager_thread
@@ -127,7 +127,6 @@ def default_init_params_workflow(driver: Driver, context: Context) -> None:
                     message_type=MessageTypeLegacy.GET_PARAMETERS,
                     dst_node_id=random_client.node_id,
                     group_id="0",
-                    ttl=DEFAULT_TTL,
                 )
             ]
         )
@@ -226,7 +225,6 @@ def default_fit_workflow(  # pylint: disable=R0914
             message_type=MessageType.TRAIN,
             dst_node_id=proxy.node_id,
             group_id=str(current_round),
-            ttl=DEFAULT_TTL,
         )
         for proxy, fitins in client_instructions
     ]
@@ -306,7 +304,6 @@ def default_evaluate_workflow(driver: Driver, context: Context) -> None:
             message_type=MessageType.EVALUATE,
             dst_node_id=proxy.node_id,
             group_id=str(current_round),
-            ttl=DEFAULT_TTL,
         )
         for proxy, evalins in client_instructions
     ]