flwr 1.24.0__py3-none-any.whl → 1.26.0__py3-none-any.whl

flwr/superlink/servicer/control/control_servicer.py

@@ -16,17 +16,21 @@
 
 
 import hashlib
+import json
 import time
 from collections.abc import Generator, Sequence
 from logging import ERROR, INFO
 from typing import Any, cast
 
 import grpc
+import requests
 
 from flwr.cli.config_utils import get_fab_metadata
 from flwr.common import Context, RecordDict, now
 from flwr.common.constant import (
     FAB_MAX_SIZE,
+    FEDERATION_NOT_FOUND_MESSAGE,
+    FEDERATION_NOT_SPECIFIED_MESSAGE,
     HEARTBEAT_DEFAULT_INTERVAL,
     LOG_STREAM_INTERVAL,
     NO_ACCOUNT_AUTH_MESSAGE,
@@ -36,6 +40,8 @@ from flwr.common.constant import (
     PUBLIC_KEY_NOT_VALID,
     PULL_UNFINISHED_RUN_MESSAGE,
     RUN_ID_NOT_FOUND_MESSAGE,
+    SUPERLINK_NODE_ID,
+    TRANSPORT_TYPE_GRPC_ADAPTER,
     Status,
     SubStatus,
 )
@@ -76,9 +82,11 @@ from flwr.proto.control_pb2 import (  # pylint: disable=E0611
 from flwr.proto.federation_pb2 import Federation  # pylint: disable=E0611
 from flwr.proto.node_pb2 import NodeInfo  # pylint: disable=E0611
 from flwr.server.superlink.linkstate import LinkState, LinkStateFactory
+from flwr.supercore.constant import NOOP_FEDERATION, PLATFORM_API_URL
 from flwr.supercore.ffs import FfsFactory
 from flwr.supercore.object_store import ObjectStore, ObjectStoreFactory
 from flwr.supercore.primitives.asymmetric import bytes_to_public_key, uses_nist_ec_curve
+from flwr.supercore.utils import parse_app_spec, request_download_link
 from flwr.superlink.artifact_provider import ArtifactProvider
 from flwr.superlink.auth_plugin import ControlAuthnPlugin
 
@@ -96,6 +104,7 @@ class ControlServicer(control_pb2_grpc.ControlServicer):
         is_simulation: bool,
         authn_plugin: ControlAuthnPlugin,
         artifact_provider: ArtifactProvider | None = None,
+        fleet_api_type: str | None = None,
     ) -> None:
         self.linkstate_factory = linkstate_factory
         self.ffs_factory = ffs_factory
@@ -103,8 +112,9 @@ class ControlServicer(control_pb2_grpc.ControlServicer):
         self.is_simulation = is_simulation
         self.authn_plugin = authn_plugin
         self.artifact_provider = artifact_provider
+        self.fleet_api_type = fleet_api_type
 
-    def StartRun(  # pylint: disable=too-many-locals
+    def StartRun(  # pylint: disable=too-many-locals, too-many-branches, too-many-statements
         self, request: StartRunRequest, context: grpc.ServicerContext
     ) -> StartRunResponse:
         """Create run ID."""
@@ -112,7 +122,15 @@ class ControlServicer(control_pb2_grpc.ControlServicer):
         state = self.linkstate_factory.state()
         ffs = self.ffs_factory.ffs()
 
-        if len(request.fab.content) > FAB_MAX_SIZE:
+        verification_dict: dict[str, str] = {}
+        if request.app_spec:
+            fab_file, verification_dict = _get_remote_fab(
+                self.fleet_api_type, request.app_spec, context
+            )
+        else:
+            fab_file = request.fab.content
+
+        if len(fab_file) > FAB_MAX_SIZE:
             log(
                 ERROR,
                 "FAB size exceeds maximum allowed size of %d bytes.",
@@ -124,7 +142,6 @@ class ControlServicer(control_pb2_grpc.ControlServicer):
         flwr_aid = _check_flwr_aid_exists(flwr_aid, context)
         override_config = user_config_from_proto(request.override_config)
         federation_options = config_record_from_proto(request.federation_options)
-        fab_file = request.fab.content
 
         try:
             # Check that num-supernodes is set
@@ -134,10 +151,11 @@ class ControlServicer(control_pb2_grpc.ControlServicer):
                 )
 
             # Check (1) federation exists and (2) the flwr_aid is a member
-            federation = request.federation
-
+            federation = request.federation or NOOP_FEDERATION
             if not state.federation_manager.exists(federation):
-                raise ValueError(f"Federation '{federation}' does not exist.")
+                if request.federation:
+                    raise ValueError(FEDERATION_NOT_FOUND_MESSAGE % federation)
+                raise ValueError(FEDERATION_NOT_SPECIFIED_MESSAGE)
 
             if not state.federation_manager.has_member(flwr_aid, federation):
                 raise ValueError(
@@ -150,11 +168,12 @@ class ControlServicer(control_pb2_grpc.ControlServicer):
             fab = Fab(
                 hashlib.sha256(fab_file).hexdigest(),
                 fab_file,
-                dict(request.fab.verifications),
+                verification_dict,
             )
-            fab_hash = ffs.put(fab.content, {})
+            fab_hash = ffs.put(fab.content, fab.verifications)
+
             if fab_hash != fab.hash_str:
-                raise RuntimeError(
+                raise ValueError(
                     f"FAB ({fab.hash_str}) hash from request doesn't match contents"
                 )
             fab_id, fab_version = get_fab_metadata(fab.content)
@@ -164,7 +183,7 @@ class ControlServicer(control_pb2_grpc.ControlServicer):
                 fab_version,
                 fab_hash,
                 override_config,
-                request.federation,
+                federation,
                 federation_options,
                 flwr_aid,
             )
@@ -180,7 +199,7 @@ class ControlServicer(control_pb2_grpc.ControlServicer):
             # Create an empty context for the Run
             context = Context(
                 run_id=run_id,
-                node_id=0,
+                node_id=SUPERLINK_NODE_ID,
                 # Dict is invariant in mypy
                 node_config=node_config,  # type: ignore[arg-type]
                 state=RecordDict(),
@@ -190,13 +209,9 @@ class ControlServicer(control_pb2_grpc.ControlServicer):
             # Register the context at the LinkState
             state.set_serverapp_context(run_id=run_id, context=context)
 
-        # pylint: disable-next=broad-except
-        except Exception as e:
+        except ValueError as e:
             log(ERROR, "Could not start run: %s", str(e))
-            context.abort(
-                grpc.StatusCode.FAILED_PRECONDITION,
-                str(e),
-            )
+            context.abort(grpc.StatusCode.FAILED_PRECONDITION, str(e))
 
         log(INFO, "Created run %s", str(run_id))
         return StartRunResponse(run_id=run_id)
@@ -521,9 +536,10 @@ class ControlServicer(control_pb2_grpc.ControlServicer):
 
         # Get federations the account is a member of
         federations = state.federation_manager.get_federations(flwr_aid=flwr_aid)
-
         return ListFederationsResponse(
-            federations=[Federation(name=fed) for fed in federations]
+            federations=[
+                Federation(name=fed[0], description=fed[1]) for fed in federations
+            ]
         )
 
     def ShowFederation(
@@ -543,7 +559,7 @@ class ControlServicer(control_pb2_grpc.ControlServicer):
 
         # Ensure flwr_aid is a member of the requested federation
         federation = request.federation_name
-        if federation not in federations:
+        if federation not in [fed[0] for fed in federations]:
             context.abort(
                 grpc.StatusCode.FAILED_PRECONDITION,
                 f"Federation '{federation}' does not exist or you are "
@@ -556,7 +572,8 @@ class ControlServicer(control_pb2_grpc.ControlServicer):
         # Build Federation proto object
         federation_proto = Federation(
             name=federation,
-            member_aids=details.member_aids,
+            member_aids=[acc.id for acc in details.accounts],  # Deprecated in v1.26.0
+            accounts=details.accounts,
             nodes=details.nodes,
             runs=[run_to_proto(run) for run in details.runs],
         )
@@ -612,3 +629,71 @@ def _check_flwr_aid_in_run(
             grpc.StatusCode.PERMISSION_DENIED,
             "⛔️ Run ID does not belong to the account",
         )
+
+
+def _format_verification(verifications: list[dict[str, str]]) -> dict[str, str]:
+    """Format verification information for FAB."""
+    # Convert verifications to dict[str, str] type
+    verification_dict = {
+        item["public_key_id"]: json.dumps(
+            {k: v for k, v in item.items() if k != "public_key_id"}
+        )
+        for item in verifications
+    }
+    verification_dict.update({"valid_license": "Valid"})
+
+    return verification_dict
+
+
+def _get_remote_fab(
+    fleet_api_type: str | None,
+    app_spec: str,
+    context: grpc.ServicerContext,
+) -> tuple[bytes, dict[str, str]]:
+    """Get remote FAB from Flower platform API."""
+    if fleet_api_type == TRANSPORT_TYPE_GRPC_ADAPTER:
+        context.abort(
+            grpc.StatusCode.FAILED_PRECONDITION,
+            "The selected SuperLink transport type is not "
+            "supported for connecting to Flower Platform.",
+        )
+
+    # Parse and validate app specification
+    try:
+        app_id, app_version = parse_app_spec(app_spec)
+    except ValueError as e:
+        context.abort(
+            grpc.StatusCode.FAILED_PRECONDITION,
+            f"{e}",
+        )
+
+    # Request download link and verification information
+    url = f"{PLATFORM_API_URL}/hub/fetch-fab"
+    try:
+        presigned_url, verifications = request_download_link(
+            app_id, app_version, url, "fab_url"
+        )
+    except ValueError as e:
+        context.abort(
+            grpc.StatusCode.FAILED_PRECONDITION,
+            f"{e}",
+        )
+
+    # Format verification information
+    verification_dict = (
+        _format_verification(verifications)
+        if verifications is not None
+        else {"valid_license": ""}
+    )
+
+    # Download FAB from Flower platform API
+    try:
+        r = requests.get(presigned_url, timeout=60)
+        r.raise_for_status()
+    except requests.RequestException as e:
+        context.abort(
+            grpc.StatusCode.FAILED_PRECONDITION,
+            f"FAB download failed: {str(e)}",
+        )
+    fab_file = r.content
+    return fab_file, verification_dict

flwr/supernode/cli/flower_supernode.py

@@ -233,7 +233,7 @@ def _try_setup_client_authentication(
 
     try:
         ssh_private_key = load_ssh_private_key(
-            Path(args.auth_supernode_private_key).read_bytes(),
+            Path(args.auth_supernode_private_key).expanduser().read_bytes(),
             None,
         )
         if not isinstance(ssh_private_key, ec.EllipticCurvePrivateKey):
@@ -260,6 +260,7 @@ def _try_obtain_trusted_entities(
     """Validate and return the trust entities."""
     if not trusted_entities_path:
         return None
+    trusted_entities_path = trusted_entities_path.expanduser()
     if not trusted_entities_path.is_file():
         flwr_exit(
             ExitCode.SUPERNODE_INVALID_TRUSTED_ENTITIES,

flwr/supernode/nodestate/in_memory_nodestate.py

@@ -19,7 +19,7 @@ from collections.abc import Sequence
 from dataclasses import dataclass
 from threading import Lock, RLock
 
-from flwr.common import Context, Error, Message
+from flwr.common import Context, Error, Message, now
 from flwr.common.constant import ErrorCode
 from flwr.common.inflatable import (
     get_all_nested_objects,
@@ -27,6 +27,7 @@ from flwr.common.inflatable import (
     no_object_id_recompute,
 )
 from flwr.common.typing import Run
+from flwr.supercore.constant import MESSAGE_TIME_ENTRY_MAX_AGE_SECONDS
 from flwr.supercore.corestate.in_memory_corestate import InMemoryCoreState
 from flwr.supercore.object_store import ObjectStore
 
@@ -45,6 +46,14 @@ class MessageEntry:
     is_retrieved: bool = False
 
 
+@dataclass
+class TimeEntry:
+    """Data class to represent a time entry."""
+
+    starting_at: float
+    finished_at: float | None = None
+
+
 class InMemoryNodeState(
     NodeState, InMemoryCoreState
 ):  # pylint: disable=too-many-instance-attributes
@@ -63,6 +72,9 @@ class InMemoryNodeState(
         # Store run ID to Context mapping
         self.ctx_store: dict[int, Context] = {}
         self.lock_ctx_store = Lock()
+        # Store msg ID to TimeEntry mapping
+        self.time_store: dict[str, TimeEntry] = {}
+        self.lock_time_store = Lock()
 
     def set_node_id(self, node_id: int | None) -> None:
         """Set the node ID."""
@@ -208,3 +220,52 @@ class InMemoryNodeState(
 
                 # Store the error reply message
                 self.store_message(error_reply)
+
+    def record_message_processing_start(self, message_id: str) -> None:
+        """Record the start time of message processing based on the message ID."""
+        with self.lock_time_store:
+            self.time_store[message_id] = TimeEntry(starting_at=now().timestamp())
+
+    def record_message_processing_end(self, message_id: str) -> None:
+        """Record the end time of message processing based on the message ID."""
+        with self.lock_time_store:
+            if message_id not in self.time_store:
+                raise ValueError(
+                    f"Cannot record end time: Message ID {message_id} not found."
+                )
+            entry = self.time_store[message_id]
+            entry.finished_at = now().timestamp()
+
+    def get_message_processing_duration(self, message_id: str) -> float:
+        """Get the message processing duration based on the message ID."""
+        # Cleanup old message processing times
+        self._cleanup_old_message_times()
+        with self.lock_time_store:
+            if message_id not in self.time_store:
+                raise ValueError(f"Message ID {message_id} not found.")
+
+            entry = self.time_store[message_id]
+            if entry.starting_at is None or entry.finished_at is None:
+                raise ValueError(
+                    f"Start time or end time for message ID {message_id} is missing."
+                )
+
+            duration = entry.finished_at - entry.starting_at
+            return duration
+
+    def _cleanup_old_message_times(self) -> None:
+        """Remove time entries older than MESSAGE_TIME_ENTRY_MAX_AGE_SECONDS."""
+        with self.lock_time_store:
+            cutoff = now().timestamp() - MESSAGE_TIME_ENTRY_MAX_AGE_SECONDS
+            # Find message IDs for entries that have a finishing_at time
+            # before the cutoff, and those that don't exist in msg_store
+            to_delete = [
+                msg_id
+                for msg_id, entry in self.time_store.items()
+                if (entry.finished_at and entry.finished_at < cutoff)
+                or msg_id not in self.msg_store
+            ]
+
+            # Delete the identified entries
+            for msg_id in to_delete:
+                del self.time_store[msg_id]

flwr/supernode/nodestate/nodestate.py

@@ -168,3 +168,48 @@ class NodeState(CoreState):
         Sequence[int]
             Sequence of run IDs with pending messages.
         """
+
+    @abstractmethod
+    def record_message_processing_start(self, message_id: str) -> None:
+        """Record the start time of message processing based on the message ID.
+
+        Parameters
+        ----------
+        message_id : str
+            The ID of the message associated with the start time.
+        """
+
+    @abstractmethod
+    def record_message_processing_end(self, message_id: str) -> None:
+        """Record the end time of message processing based on the message ID.
+
+        Parameters
+        ----------
+        message_id : str
+            The ID of the message associated with the end time.
+
+        Raises
+        ------
+        ValueError
+            If the message ID is not found.
+        """
+
+    @abstractmethod
+    def get_message_processing_duration(self, message_id: str) -> float:
+        """Get the message processing duration based on the message ID.
+
+        Parameters
+        ----------
+        message_id : str
+            The ID of the message.
+
+        Returns
+        -------
+        float
+            The processing duration in seconds.
+
+        Raises
+        ------
+        ValueError
+            If the message ID is not found, or if start/end times are missing.
+        """

flwr/supernode/runtime/run_clientapp.py

@@ -41,7 +41,7 @@ from flwr.common.inflatable_protobuf_utils import (
 from flwr.common.inflatable_utils import pull_and_inflate_object_from_tree, push_objects
 from flwr.common.logger import log
 from flwr.common.message import remove_content_from_message
-from flwr.common.retry_invoker import _make_simple_grpc_retry_invoker, _wrap_stub
+from flwr.common.retry_invoker import make_simple_grpc_retry_invoker, wrap_stub
 from flwr.common.serde import (
     context_from_proto,
     context_to_proto,
@@ -103,14 +103,14 @@ def run_clientapp(  # pylint: disable=R0913, R0914, R0917
     flwr_dir_ = get_flwr_dir(flwr_dir)
     try:
         stub = ClientAppIoStub(channel)
-        _wrap_stub(stub, _make_simple_grpc_retry_invoker())
+        wrap_stub(stub, make_simple_grpc_retry_invoker())
 
         # Start app heartbeat
         heartbeat_sender = HeartbeatSender(make_app_heartbeat_fn_grpc(stub, token))
         heartbeat_sender.start()
 
         # Pull Message, Context, Run and (optional) FAB from SuperNode
-        message, context, run, fab = pull_clientappinputs(stub=stub, token=token)
+        message, context, run, fab = pull_appinputs(stub=stub, token=token)
 
         try:
 
@@ -152,7 +152,7 @@ def run_clientapp(  # pylint: disable=R0913, R0914, R0917
             reply_message = Message(Error(code=e_code, reason=reason), reply_to=message)
 
         # Push Message and Context to SuperNode
-        _ = push_clientappoutputs(
+        _ = push_appoutputs(
             stub=stub, token=token, message=reply_message, context=context
         )
 
@@ -165,15 +165,15 @@ def run_clientapp(  # pylint: disable=R0913, R0914, R0917
     )
 
 
-def pull_clientappinputs(
+def pull_appinputs(
     stub: ClientAppIoStub, token: str
 ) -> tuple[Message, Context, Run, Fab | None]:
-    """Pull ClientAppInputs from SuperNode."""
+    """Pull AppInputs from SuperNode."""
     masked_token = mask_string(token)
-    log(INFO, "[flwr-clientapp] Pull `ClientAppInputs` for token %s", masked_token)
+    log(INFO, "[flwr-clientapp] Pull `AppInputs` for token %s", masked_token)
     try:
         # Pull Context, Run and (optional) FAB
-        res: PullAppInputsResponse = stub.PullClientAppInputs(
+        res: PullAppInputsResponse = stub.PullAppInputs(
             PullAppInputsRequest(token=token)
         )
         context = context_from_proto(res.context)
@@ -201,16 +201,16 @@ def pull_clientappinputs(
         message.metadata.__dict__["_message_id"] = object_tree.object_id
         return message, context, run, fab
     except grpc.RpcError as e:
-        log(ERROR, "[PullClientAppInputs] gRPC error occurred: %s", str(e))
+        log(ERROR, "[PullAppInputs] gRPC error occurred: %s", str(e))
         raise e
 
 
-def push_clientappoutputs(
+def push_appoutputs(
     stub: ClientAppIoStub, token: str, message: Message, context: Context
 ) -> PushAppOutputsResponse:
-    """Push ClientAppOutputs to SuperNode."""
+    """Push AppOutputs to SuperNode."""
     masked_token = mask_string(token)
-    log(INFO, "[flwr-clientapp] Push `ClientAppOutputs` for token %s", masked_token)
+    log(INFO, "[flwr-clientapp] Push `AppOutputs` for token %s", masked_token)
     # Set message ID
     message.metadata.__dict__["_message_id"] = message.object_id
     proto_message = message_to_proto(remove_content_from_message(message))
@@ -250,10 +250,10 @@ def push_clientappoutputs(
             )
 
         # Push Context
-        res: PushAppOutputsResponse = stub.PushClientAppOutputs(
+        res: PushAppOutputsResponse = stub.PushAppOutputs(
             PushAppOutputsRequest(token=token, context=proto_context)
         )
         return res
     except grpc.RpcError as e:
-        log(ERROR, "[PushClientAppOutputs] gRPC error occurred: %s", str(e))
+        log(ERROR, "[PushAppOutputs] gRPC error occurred: %s", str(e))
         raise e

flwr/supernode/servicer/clientappio/clientappio_servicer.py

@@ -129,11 +129,11 @@ class ClientAppIoServicer(clientappio_pb2_grpc.ClientAppIoServicer):
 
         return GetRunResponse(run=run_to_proto(run))
 
-    def PullClientAppInputs(
+    def PullAppInputs(
         self, request: PullAppInputsRequest, context: grpc.ServicerContext
     ) -> PullAppInputsResponse:
         """Pull Message, Context, and Run."""
-        log(DEBUG, "ClientAppIo.PullClientAppInputs")
+        log(DEBUG, "ClientAppIo.PullAppInputs")
 
         # Initialize state and ffs connection
         state = self.state_factory.state()
@@ -176,11 +176,11 @@ class ClientAppIoServicer(clientappio_pb2_grpc.ClientAppIoServicer):
             fab=fab_to_proto(fab),
         )
 
-    def PushClientAppOutputs(
+    def PushAppOutputs(
         self, request: PushAppOutputsRequest, context: grpc.ServicerContext
     ) -> PushAppOutputsResponse:
         """Push Message and Context."""
-        log(DEBUG, "ClientAppIo.PushClientAppOutputs")
+        log(DEBUG, "ClientAppIo.PushAppOutputs")
 
         # Initialize state connection
         state = self.state_factory.state()
@@ -223,6 +223,9 @@ class ClientAppIoServicer(clientappio_pb2_grpc.ClientAppIoServicer):
         # Retrieve message for this run
         message = state.get_messages(run_ids=[run_id], is_reply=False)[0]
 
+        # Record message processing start time
+        state.record_message_processing_start(message_id=message.metadata.message_id)
+
         # Retrieve the object tree for the message
         object_tree = store.get_object_tree(message.metadata.message_id)
 
@@ -248,13 +251,18 @@ class ClientAppIoServicer(clientappio_pb2_grpc.ClientAppIoServicer):
             )
             raise RuntimeError("This line should never be reached.")
 
+        # Record message processing end time
+        state.record_message_processing_end(
+            message_id=request.messages_list[0].metadata.reply_to_message_id
+        )
+
         # Store Message object to descendants mapping and preregister objects
         objects_to_push: set[str] = set()
         for object_tree in request.message_object_trees:
             objects_to_push |= set(store.preregister(run_id, object_tree))
+
         # Save the message to the state
         state.store_message(message_from_proto(request.messages_list[0]))
-
         return PushAppMessagesResponse(objects_to_push=objects_to_push)
 
     def SendAppHeartbeat(

flwr/supernode/start_client_internal.py

@@ -32,10 +32,10 @@ from cryptography.hazmat.primitives.asymmetric import ec, ed25519
 from cryptography.hazmat.primitives.serialization.ssh import load_ssh_public_key
 from grpc import RpcError
 
+from flwr.app.user_config import UserConfig
 from flwr.client.grpc_adapter_client.connection import grpc_adapter
 from flwr.client.grpc_rere_client.connection import grpc_request_response
 from flwr.common import GRPC_MAX_MESSAGE_LENGTH, Context, Error, Message, RecordDict
-from flwr.common.address import parse_address
 from flwr.common.config import get_flwr_dir, get_fused_config_from_fab
 from flwr.common.constant import (
     CLIENTAPPIO_API_DEFAULT_SERVER_ADDRESS,
@@ -60,12 +60,12 @@ from flwr.common.inflatable_utils import (
     push_object_contents_from_iterable,
 )
 from flwr.common.logger import log
-from flwr.common.retry_invoker import RetryInvoker, _make_simple_grpc_retry_invoker
+from flwr.common.retry_invoker import RetryInvoker, make_simple_grpc_retry_invoker
 from flwr.common.telemetry import EventType
-from flwr.common.typing import Fab, Run, RunNotRunningException, UserConfig
-from flwr.common.version import package_version
+from flwr.common.typing import Fab, Run, RunNotRunningException
 from flwr.proto.clientappio_pb2_grpc import add_ClientAppIoServicer_to_server
 from flwr.proto.message_pb2 import ObjectTree  # pylint: disable=E0611
+from flwr.supercore.address import parse_address, resolve_bind_address
 from flwr.supercore.ffs import Ffs, FfsFactory
 from flwr.supercore.grpc_health import run_health_server_grpc_no_tls
 from flwr.supercore.object_store import ObjectStore, ObjectStoreFactory
@@ -74,6 +74,7 @@ from flwr.supercore.primitives.asymmetric_ed25519 import (
     decode_base64url,
     verify_signature,
 )
+from flwr.supercore.version import package_version
 from flwr.supernode.nodestate import NodeState, NodeStateFactory
 from flwr.supernode.servicer.clientappio import ClientAppIoServicer
 
@@ -212,7 +213,10 @@ def start_client_internal(
     # Launch the SuperExec if the isolation mode is `subprocess`
     if isolation == ISOLATION_MODE_SUBPROCESS:
         command = ["flower-superexec", "--insecure"]
-        command += ["--appio-api-address", clientappio_api_address]
+        command += [
+            "--appio-api-address",
+            resolve_bind_address(clientappio_api_address),
+        ]
         command += ["--plugin-type", ExecPluginType.CLIENT_APP]
         command += ["--parent-pid", str(os.getpid())]
         # pylint: disable-next=consider-using-with
@@ -432,7 +436,7 @@ def _pull_and_store_message(  # pylint: disable=too-many-positional-arguments
 def _push_messages(
     state: NodeState,
     object_store: ObjectStore,
-    send: Callable[[Message, ObjectTree], set[str]],
+    send: Callable[[Message, ObjectTree, float], set[str]],
     push_object: Callable[[int, str, bytes], None],
 ) -> None:
     """Push reply messages to the SuperLink."""
@@ -480,9 +484,12 @@ def _push_messages(
 
         # Send the message
         try:
-            # Send the reply message with its ObjectTree
+            clientapp_runtime = state.get_message_processing_duration(
+                message_id=message.metadata.reply_to_message_id,
+            )
+            # Send the reply message with its ObjectTree and ClientApp runtime
             # Get the IDs of objects to send
-            ids_obj_to_send = send(message, object_tree)
+            ids_obj_to_send = send(message, object_tree, clientapp_runtime)
 
             # Push object contents from the ObjectStore
             run_id = message.metadata.run_id
@@ -539,7 +546,7 @@ def _init_connection(  # pylint: disable=too-many-positional-arguments
     tuple[
         int,
         Callable[[], tuple[Message, ObjectTree] | None],
-        Callable[[Message, ObjectTree], set[str]],
+        Callable[[Message, ObjectTree, float], set[str]],
         Callable[[int], Run],
         Callable[[str, int], Fab],
         Callable[[int, str], bytes],
@@ -603,7 +610,7 @@ def _make_fleet_connection_retry_invoker(
     connection_error_type: type[Exception] = RpcError,
 ) -> RetryInvoker:
     """Create a retry invoker for fleet connection."""
-    retry_invoker = _make_simple_grpc_retry_invoker()
+    retry_invoker = make_simple_grpc_retry_invoker()
     retry_invoker.recoverable_exceptions = connection_error_type
     if max_retries is not None:
         retry_invoker.max_tries = max_retries + 1