flwr-nightly 1.17.0.dev20250318__py3-none-any.whl → 1.17.0.dev20250320__py3-none-any.whl

flwr/server/superlink/linkstate/utils.py

@@ -27,19 +27,14 @@ from flwr.common.constant import (
     Status,
     SubStatus,
 )
+from flwr.common.message import make_message
 from flwr.common.typing import RunStatus
 
 # pylint: disable=E0611
 from flwr.proto.message_pb2 import Context as ProtoContext
-from flwr.proto.recordset_pb2 import ConfigsRecord as ProtoConfigsRecord
+from flwr.proto.recorddict_pb2 import ConfigsRecord as ProtoConfigsRecord
 
 # pylint: enable=E0611
-
-NODE_UNAVAILABLE_ERROR_REASON = (
-    "Error: Node Unavailable - The destination node is currently unavailable. "
-    "It exceeds the time limit specified in its last ping."
-)
-
 VALID_RUN_STATUS_TRANSITIONS = {
     (Status.PENDING, Status.STARTING),
     (Status.STARTING, Status.RUNNING),
@@ -60,6 +55,10 @@ MESSAGE_UNAVAILABLE_ERROR_REASON = (
 REPLY_MESSAGE_UNAVAILABLE_ERROR_REASON = (
     "Error: Reply Message Unavailable - The reply message has expired."
 )
+NODE_UNAVAILABLE_ERROR_REASON = (
+    "Error: Node Unavailable - The destination node is currently unavailable. "
+    "It exceeds twice the time limit specified in its last ping."
+)
 
 
 def generate_rand_int_from_bytes(
@@ -237,7 +236,9 @@ def has_valid_sub_status(status: RunStatus) -> bool:
     return status.sub_status == ""
 
 
-def create_message_error_unavailable_res_message(ins_metadata: Metadata) -> Message:
+def create_message_error_unavailable_res_message(
+    ins_metadata: Metadata, error_type: str
+) -> Message:
     """Generate an error Message that the SuperLink returns carrying the specified
     error."""
     current_time = now().timestamp()
@@ -247,22 +248,31 @@ def create_message_error_unavailable_res_message(ins_metadata: Metadata) -> Mess
         message_id=str(uuid4()),
         src_node_id=SUPERLINK_NODE_ID,
         dst_node_id=SUPERLINK_NODE_ID,
-        reply_to_message=ins_metadata.message_id,
+        reply_to_message_id=ins_metadata.message_id,
         group_id=ins_metadata.group_id,
         message_type=ins_metadata.message_type,
+        created_at=current_time,
         ttl=ttl,
     )
 
-    return Message(
+    return make_message(
         metadata=metadata,
         error=Error(
-            code=ErrorCode.REPLY_MESSAGE_UNAVAILABLE,
-            reason=REPLY_MESSAGE_UNAVAILABLE_ERROR_REASON,
+            code=(
+                ErrorCode.REPLY_MESSAGE_UNAVAILABLE
+                if error_type == "msg_unavail"
+                else ErrorCode.NODE_UNAVAILABLE
+            ),
+            reason=(
+                REPLY_MESSAGE_UNAVAILABLE_ERROR_REASON
+                if error_type == "msg_unavail"
+                else NODE_UNAVAILABLE_ERROR_REASON
+            ),
         ),
     )
 
 
-def create_message_error_unavailable_ins_message(reply_to_message: UUID) -> Message:
+def create_message_error_unavailable_ins_message(reply_to_message_id: UUID) -> Message:
     """Error to indicate that the enquired Message had expired before reply arrived or
     that it isn't found."""
     metadata = Metadata(
@@ -270,13 +280,14 @@ def create_message_error_unavailable_ins_message(reply_to_message: UUID) -> Mess
         message_id=str(uuid4()),
         src_node_id=SUPERLINK_NODE_ID,
         dst_node_id=SUPERLINK_NODE_ID,
-        reply_to_message=str(reply_to_message),
+        reply_to_message_id=str(reply_to_message_id),
         group_id="",  # Unknown
         message_type=MessageType.SYSTEM,
+        created_at=now().timestamp(),
         ttl=0,
     )
 
-    return Message(
+    return make_message(
         metadata=metadata,
         error=Error(
             code=ErrorCode.MESSAGE_UNAVAILABLE,
@@ -364,14 +375,63 @@ def verify_found_message_replies(
     ret_dict: dict[UUID, Message] = {}
     current = current_time if current_time else now().timestamp()
     for message_res in found_message_res_list:
-        message_ins_id = UUID(message_res.metadata.reply_to_message)
+        message_ins_id = UUID(message_res.metadata.reply_to_message_id)
         if update_set:
             inquired_message_ids.remove(message_ins_id)
         # Check if the reply Message has expired
         if message_ttl_has_expired(message_res.metadata, current):
             # No need to insert the error Message
             message_res = create_message_error_unavailable_res_message(
-                found_message_ins_dict[message_ins_id].metadata
+                found_message_ins_dict[message_ins_id].metadata, "msg_unavail"
             )
         ret_dict[message_ins_id] = message_res
     return ret_dict
+
+
+def check_node_availability_for_in_message(
+    inquired_in_message_ids: set[UUID],
+    found_in_message_dict: dict[UUID, Message],
+    node_id_to_online_until: dict[int, float],
+    current_time: Optional[float] = None,
+    update_set: bool = True,
+) -> dict[UUID, Message]:
+    """Check node availability for given Message and generate error reply Message if
+    unavailable. A Message error indicating node unavailability will be generated for
+    each given Message whose destination node is offline or non-existent.
+
+    Parameters
+    ----------
+    inquired_in_message_ids : set[UUID]
+        Set of Message IDs for which to check destination node availability.
+    found_in_message_dict : dict[UUID, Message]
+        Dictionary containing all found Message indexed by their IDs.
+    node_id_to_online_until : dict[int, float]
+        Dictionary mapping node IDs to their online-until timestamps.
+    current_time : Optional[float] (default: None)
+        The current time to check for expiration. If set to `None`, the current time
+        will automatically be set to the current timestamp using `now().timestamp()`.
+    update_set : bool (default: True)
+        If True, the `inquired_in_message_ids` will be updated to remove invalid ones,
+        by default True.
+
+    Returns
+    -------
+    dict[UUID, Message]
+        A dictionary of error Message indexed by the corresponding Message ID.
+    """
+    ret_dict = {}
+    current = current_time if current_time else now().timestamp()
+    for in_message_id in list(inquired_in_message_ids):
+        in_message = found_in_message_dict[in_message_id]
+        node_id = in_message.metadata.dst_node_id
+        online_until = node_id_to_online_until.get(node_id)
+        # Generate a reply message containing an error reply
+        # if the node is offline or doesn't exist.
+        if online_until is None or online_until < current:
+            if update_set:
+                inquired_in_message_ids.remove(in_message_id)
+            reply_message = create_message_error_unavailable_res_message(
+                in_message.metadata, "node_unavail"
+            )
+            ret_dict[in_message_id] = reply_message
+    return ret_dict

flwr/server/superlink/serverappio/serverappio_servicer.py

@@ -206,7 +206,7 @@ class ServerAppIoServicer(serverappio_pb2_grpc.ServerAppIoServicer):
 
         # Delete the instruction Messages and their replies if found
         message_ins_ids_to_delete = {
-            UUID(msg_res.metadata.reply_to_message) for msg_res in messages_res
+            UUID(msg_res.metadata.reply_to_message_id) for msg_res in messages_res
         }
 
         state.delete_messages(message_ins_ids=message_ins_ids_to_delete)

flwr/server/typing.py

@@ -19,9 +19,9 @@ from typing import Callable
 
 from flwr.common import Context
 
-from .grid import Driver
+from .grid import Grid
 from .serverapp_components import ServerAppComponents
 
-ServerAppCallable = Callable[[Driver, Context], None]
-Workflow = Callable[[Driver, Context], None]
+ServerAppCallable = Callable[[Grid, Context], None]
+Workflow = Callable[[Grid, Context], None]
 ServerFn = Callable[[Context], ServerAppComponents]

flwr/server/utils/validator.py

@@ -68,8 +68,8 @@ def validate_message(message: Message, is_reply_message: bool) -> list[str]:
 
     # Link respose to original message
     if not is_reply_message:
-        if metadata.reply_to_message != "":
-            validation_errors.append("`metadata.reply_to_message` MUST not be set.")
+        if metadata.reply_to_message_id != "":
+            validation_errors.append("`metadata.reply_to_message_id` MUST not be set.")
         if metadata.src_node_id != SUPERLINK_NODE_ID:
             validation_errors.append(
                 f"`metadata.src_node_id` is not {SUPERLINK_NODE_ID} (SuperLink node ID)"
@@ -79,8 +79,8 @@ def validate_message(message: Message, is_reply_message: bool) -> list[str]:
                 f"`metadata.dst_node_id` is {SUPERLINK_NODE_ID} (SuperLink node ID)"
             )
     else:
-        if metadata.reply_to_message == "":
-            validation_errors.append("`metadata.reply_to_message` MUST be set.")
+        if metadata.reply_to_message_id == "":
+            validation_errors.append("`metadata.reply_to_message_id` MUST be set.")
         if metadata.src_node_id == SUPERLINK_NODE_ID:
             validation_errors.append(
                 f"`metadata.src_node_id` is {SUPERLINK_NODE_ID} (SuperLink node ID)"

flwr/server/workflow/default_workflows.py

@@ -20,7 +20,7 @@ import timeit
 from logging import INFO, WARN
 from typing import Optional, Union, cast
 
-import flwr.common.recordset_compat as compat
+import flwr.common.recorddict_compat as compat
 from flwr.common import (
     Code,
     ConfigsRecord,
@@ -36,7 +36,7 @@ from flwr.common.constant import MessageType, MessageTypeLegacy
 from ..client_proxy import ClientProxy
 from ..compat.app_utils import start_update_client_manager_thread
 from ..compat.legacy_context import LegacyContext
-from ..grid import Driver
+from ..grid import Grid
 from ..typing import Workflow
 from .constant import MAIN_CONFIGS_RECORD, MAIN_PARAMS_RECORD, Key
 
@@ -56,7 +56,7 @@ class DefaultWorkflow:
         self.fit_workflow: Workflow = fit_workflow
         self.evaluate_workflow: Workflow = evaluate_workflow
 
-    def __call__(self, driver: Driver, context: Context) -> None:
+    def __call__(self, grid: Grid, context: Context) -> None:
         """Execute the workflow."""
         if not isinstance(context, LegacyContext):
             raise TypeError(
@@ -65,7 +65,7 @@ class DefaultWorkflow:
 
         # Start the thread updating nodes
         thread, f_stop, c_done = start_update_client_manager_thread(
-            driver, context.client_manager
+            grid, context.client_manager
         )
 
         # Wait until the node registration done
@@ -73,7 +73,7 @@ class DefaultWorkflow:
 
         # Initialize parameters
         log(INFO, "[INIT]")
-        default_init_params_workflow(driver, context)
+        default_init_params_workflow(grid, context)
 
         # Run federated learning for num_rounds
         start_time = timeit.default_timer()
@@ -87,13 +87,13 @@ class DefaultWorkflow:
             cfg[Key.CURRENT_ROUND] = current_round
 
             # Fit round
-            self.fit_workflow(driver, context)
+            self.fit_workflow(grid, context)
 
             # Centralized evaluation
-            default_centralized_evaluation_workflow(driver, context)
+            default_centralized_evaluation_workflow(grid, context)
 
             # Evaluate round
-            self.evaluate_workflow(driver, context)
+            self.evaluate_workflow(grid, context)
 
         # Bookkeeping and log results
         end_time = timeit.default_timer()
@@ -119,7 +119,7 @@ class DefaultWorkflow:
         thread.join()
 
 
-def default_init_params_workflow(driver: Driver, context: Context) -> None:
+def default_init_params_workflow(grid: Grid, context: Context) -> None:
     """Execute the default workflow for parameters initialization."""
     if not isinstance(context, LegacyContext):
         raise TypeError(f"Expect a LegacyContext, but get {type(context).__name__}.")
@@ -137,10 +137,10 @@ def default_init_params_workflow(driver: Driver, context: Context) -> None:
         log(INFO, "Requesting initial parameters from one random client")
         random_client = context.client_manager.sample(1)[0]
         # Send GetParametersIns and get the response
-        content = compat.getparametersins_to_recordset(GetParametersIns({}))
-        messages = driver.send_and_receive(
+        content = compat.getparametersins_to_recorddict(GetParametersIns({}))
+        messages = grid.send_and_receive(
             [
-                driver.create_message(
+                grid.create_message(
                     content=content,
                     message_type=MessageTypeLegacy.GET_PARAMETERS,
                     dst_node_id=random_client.node_id,
@@ -152,7 +152,7 @@ def default_init_params_workflow(driver: Driver, context: Context) -> None:
 
         if (
             msg.has_content()
-            and compat._extract_status_from_recordset(  # pylint: disable=W0212
+            and compat._extract_status_from_recorddict(  # pylint: disable=W0212
                 "getparametersres", msg.content
             ).code
             == Code.OK
@@ -186,7 +186,7 @@ def default_init_params_workflow(driver: Driver, context: Context) -> None:
         log(INFO, "Evaluation returned no results (`None`)")
 
 
-def default_centralized_evaluation_workflow(_: Driver, context: Context) -> None:
+def default_centralized_evaluation_workflow(_: Grid, context: Context) -> None:
     """Execute the default workflow for centralized evaluation."""
     if not isinstance(context, LegacyContext):
         raise TypeError(f"Expect a LegacyContext, but get {type(context).__name__}.")
@@ -218,9 +218,7 @@ def default_centralized_evaluation_workflow(_: Driver, context: Context) -> None
         )
 
 
-def default_fit_workflow(  # pylint: disable=R0914
-    driver: Driver, context: Context
-) -> None:
+def default_fit_workflow(grid: Grid, context: Context) -> None:  # pylint: disable=R0914
     """Execute the default workflow for a single fit round."""
     if not isinstance(context, LegacyContext):
         raise TypeError(f"Expect a LegacyContext, but get {type(context).__name__}.")
@@ -255,8 +253,8 @@ def default_fit_workflow(  # pylint: disable=R0914
 
     # Build out messages
     out_messages = [
-        driver.create_message(
-            content=compat.fitins_to_recordset(fitins, True),
+        grid.create_message(
+            content=compat.fitins_to_recorddict(fitins, True),
             message_type=MessageType.TRAIN,
             dst_node_id=proxy.node_id,
             group_id=str(current_round),
@@ -266,7 +264,7 @@ def default_fit_workflow(  # pylint: disable=R0914
 
     # Send instructions to clients and
     # collect `fit` results from all clients participating in this round
-    messages = list(driver.send_and_receive(out_messages))
+    messages = list(grid.send_and_receive(out_messages))
     del out_messages
     num_failures = len([msg for msg in messages if msg.has_error()])
 
@@ -284,7 +282,7 @@ def default_fit_workflow(  # pylint: disable=R0914
     for msg in messages:
         if msg.has_content():
             proxy = node_id_to_proxy[msg.metadata.src_node_id]
-            fitres = compat.recordset_to_fitres(msg.content, False)
+            fitres = compat.recorddict_to_fitres(msg.content, False)
             if fitres.status.code == Code.OK:
                 results.append((proxy, fitres))
             else:
@@ -307,7 +305,7 @@ def default_fit_workflow(  # pylint: disable=R0914
 
 
 # pylint: disable-next=R0914
-def default_evaluate_workflow(driver: Driver, context: Context) -> None:
+def default_evaluate_workflow(grid: Grid, context: Context) -> None:
     """Execute the default workflow for a single evaluate round."""
     if not isinstance(context, LegacyContext):
         raise TypeError(f"Expect a LegacyContext, but get {type(context).__name__}.")
@@ -341,8 +339,8 @@ def default_evaluate_workflow(driver: Driver, context: Context) -> None:
 
     # Build out messages
     out_messages = [
-        driver.create_message(
-            content=compat.evaluateins_to_recordset(evalins, True),
+        grid.create_message(
+            content=compat.evaluateins_to_recorddict(evalins, True),
             message_type=MessageType.EVALUATE,
             dst_node_id=proxy.node_id,
             group_id=str(current_round),
@@ -352,7 +350,7 @@ def default_evaluate_workflow(driver: Driver, context: Context) -> None:
 
     # Send instructions to clients and
     # collect `evaluate` results from all clients participating in this round
-    messages = list(driver.send_and_receive(out_messages))
+    messages = list(grid.send_and_receive(out_messages))
     del out_messages
     num_failures = len([msg for msg in messages if msg.has_error()])
 
@@ -370,7 +368,7 @@ def default_evaluate_workflow(driver: Driver, context: Context) -> None:
     for msg in messages:
         if msg.has_content():
             proxy = node_id_to_proxy[msg.metadata.src_node_id]
-            evalres = compat.recordset_to_evaluateres(msg.content)
+            evalres = compat.recorddict_to_evaluateres(msg.content)
             if evalres.status.code == Code.OK:
                 results.append((proxy, evalres))
             else:

flwr/server/workflow/secure_aggregation/secaggplus_workflow.py

@@ -20,7 +20,7 @@ from dataclasses import dataclass, field
 from logging import DEBUG, ERROR, INFO, WARN
 from typing import Optional, Union, cast
 
-import flwr.common.recordset_compat as compat
+import flwr.common.recorddict_compat as compat
 from flwr.common import (
     ConfigsRecord,
     Context,
@@ -28,7 +28,7 @@ from flwr.common import (
     Message,
     MessageType,
     NDArrays,
-    RecordSet,
+    RecordDict,
     bytes_to_ndarray,
     log,
     ndarrays_to_parameters,
@@ -55,7 +55,7 @@ from flwr.common.secure_aggregation.secaggplus_constants import (
 from flwr.common.secure_aggregation.secaggplus_utils import pseudo_rand_gen
 from flwr.server.client_proxy import ClientProxy
 from flwr.server.compat.legacy_context import LegacyContext
-from flwr.server.grid import Driver
+from flwr.server.grid import Grid
 
 from ..constant import MAIN_CONFIGS_RECORD, MAIN_PARAMS_RECORD
 from ..constant import Key as WorkflowKey
@@ -66,7 +66,7 @@ class WorkflowState:  # pylint: disable=R0902
     """The state of the SecAgg+ protocol."""
 
     nid_to_proxies: dict[int, ClientProxy] = field(default_factory=dict)
-    nid_to_fitins: dict[int, RecordSet] = field(default_factory=dict)
+    nid_to_fitins: dict[int, RecordDict] = field(default_factory=dict)
     sampled_node_ids: set[int] = field(default_factory=set)
     active_node_ids: set[int] = field(default_factory=set)
     num_shares: int = 0
@@ -186,7 +186,7 @@ class SecAggPlusWorkflow:
 
         self._check_init_params()
 
-    def __call__(self, driver: Driver, context: Context) -> None:
+    def __call__(self, grid: Grid, context: Context) -> None:
         """Run the SecAgg+ protocol."""
         if not isinstance(context, LegacyContext):
             raise TypeError(
@@ -202,7 +202,7 @@ class SecAggPlusWorkflow:
         )
         log(INFO, "Secure aggregation commencing.")
         for step in steps:
-            if not step(driver, context, state):
+            if not step(grid, context, state):
                 log(INFO, "Secure aggregation halted.")
                 return
         log(INFO, "Secure aggregation completed.")
@@ -279,7 +279,7 @@ class SecAggPlusWorkflow:
         return True
 
     def setup_stage(  # pylint: disable=R0912, R0914, R0915
-        self, driver: Driver, context: LegacyContext, state: WorkflowState
+        self, grid: Grid, context: LegacyContext, state: WorkflowState
     ) -> bool:
         """Execute the 'setup' stage."""
         # Obtain fit instructions
@@ -303,7 +303,7 @@ class SecAggPlusWorkflow:
         )
 
         state.nid_to_fitins = {
-            proxy.node_id: compat.fitins_to_recordset(fitins, True)
+            proxy.node_id: compat.fitins_to_recorddict(fitins, True)
             for proxy, fitins in proxy_fitins_lst
         }
         state.nid_to_proxies = {proxy.node_id: proxy for proxy, _ in proxy_fitins_lst}
@@ -367,10 +367,10 @@ class SecAggPlusWorkflow:
 
         # Send setup configuration to clients
         cfgs_record = ConfigsRecord(sa_params_dict)  # type: ignore
-        content = RecordSet({RECORD_KEY_CONFIGS: cfgs_record})
+        content = RecordDict({RECORD_KEY_CONFIGS: cfgs_record})
 
         def make(nid: int) -> Message:
-            return driver.create_message(
+            return grid.create_message(
                 content=content,
                 message_type=MessageType.TRAIN,
                 dst_node_id=nid,
@@ -382,7 +382,7 @@ class SecAggPlusWorkflow:
             "[Stage 0] Sending configurations to %s clients.",
             len(state.active_node_ids),
         )
-        msgs = driver.send_and_receive(
+        msgs = grid.send_and_receive(
             [make(node_id) for node_id in state.active_node_ids], timeout=self.timeout
         )
         state.active_node_ids = {
@@ -406,7 +406,7 @@ class SecAggPlusWorkflow:
         return self._check_threshold(state)
 
     def share_keys_stage(  # pylint: disable=R0914
-        self, driver: Driver, context: LegacyContext, state: WorkflowState
+        self, grid: Grid, context: LegacyContext, state: WorkflowState
     ) -> bool:
         """Execute the 'share keys' stage."""
         cfg = context.state.configs_records[MAIN_CONFIGS_RECORD]
@@ -417,8 +417,8 @@ class SecAggPlusWorkflow:
                 {str(nid): state.nid_to_publickeys[nid] for nid in neighbours}
             )
             cfgs_record[Key.STAGE] = Stage.SHARE_KEYS
-            content = RecordSet({RECORD_KEY_CONFIGS: cfgs_record})
-            return driver.create_message(
+            content = RecordDict({RECORD_KEY_CONFIGS: cfgs_record})
+            return grid.create_message(
                 content=content,
                 message_type=MessageType.TRAIN,
                 dst_node_id=nid,
@@ -431,7 +431,7 @@ class SecAggPlusWorkflow:
             "[Stage 1] Forwarding public keys to %s clients.",
             len(state.active_node_ids),
         )
-        msgs = driver.send_and_receive(
+        msgs = grid.send_and_receive(
             [make(node_id) for node_id in state.active_node_ids], timeout=self.timeout
         )
         state.active_node_ids = {
@@ -476,7 +476,7 @@ class SecAggPlusWorkflow:
         return self._check_threshold(state)
 
     def collect_masked_vectors_stage(
-        self, driver: Driver, context: LegacyContext, state: WorkflowState
+        self, grid: Grid, context: LegacyContext, state: WorkflowState
     ) -> bool:
         """Execute the 'collect masked vectors' stage."""
         cfg = context.state.configs_records[MAIN_CONFIGS_RECORD]
@@ -491,7 +491,7 @@ class SecAggPlusWorkflow:
             cfgs_record = ConfigsRecord(cfgs_dict)  # type: ignore
             content = state.nid_to_fitins[nid]
             content.configs_records[RECORD_KEY_CONFIGS] = cfgs_record
-            return driver.create_message(
+            return grid.create_message(
                 content=content,
                 message_type=MessageType.TRAIN,
                 dst_node_id=nid,
@@ -503,7 +503,7 @@ class SecAggPlusWorkflow:
             "[Stage 2] Forwarding encrypted key shares to %s clients.",
             len(state.active_node_ids),
         )
-        msgs = driver.send_and_receive(
+        msgs = grid.send_and_receive(
             [make(node_id) for node_id in state.active_node_ids], timeout=self.timeout
         )
         state.active_node_ids = {
@@ -540,14 +540,14 @@ class SecAggPlusWorkflow:
             if msg.has_error():
                 state.failures.append(Exception(msg.error))
                 continue
-            fitres = compat.recordset_to_fitres(msg.content, True)
+            fitres = compat.recorddict_to_fitres(msg.content, True)
             proxy = state.nid_to_proxies[msg.metadata.src_node_id]
             state.legacy_results.append((proxy, fitres))
 
         return self._check_threshold(state)
 
     def unmask_stage(  # pylint: disable=R0912, R0914, R0915
-        self, driver: Driver, context: LegacyContext, state: WorkflowState
+        self, grid: Grid, context: LegacyContext, state: WorkflowState
     ) -> bool:
         """Execute the 'unmask' stage."""
         cfg = context.state.configs_records[MAIN_CONFIGS_RECORD]
@@ -566,8 +566,8 @@ class SecAggPlusWorkflow:
                 Key.DEAD_NODE_ID_LIST: list(neighbours & dead_nids),
             }
             cfgs_record = ConfigsRecord(cfgs_dict)  # type: ignore
-            content = RecordSet({RECORD_KEY_CONFIGS: cfgs_record})
-            return driver.create_message(
+            content = RecordDict({RECORD_KEY_CONFIGS: cfgs_record})
+            return grid.create_message(
                 content=content,
                 message_type=MessageType.TRAIN,
                 dst_node_id=nid,
@@ -579,7 +579,7 @@ class SecAggPlusWorkflow:
             "[Stage 3] Requesting key shares from %s clients to remove masks.",
             len(state.active_node_ids),
         )
-        msgs = driver.send_and_receive(
+        msgs = grid.send_and_receive(
             [make(node_id) for node_id in state.active_node_ids], timeout=self.timeout
         )
         state.active_node_ids = {