flwr-nightly 1.8.0.dev20240315__py3-none-any.whl → 1.15.0.dev20250114__py3-none-any.whl

flwr/client/grpc_rere_client/client_interceptor.py

@@ -0,0 +1,176 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower client interceptor."""
+
+
+import base64
+import collections
+from collections.abc import Sequence
+from logging import WARNING
+from typing import Any, Callable, Optional, Union
+
+import grpc
+from cryptography.hazmat.primitives.asymmetric import ec
+
+from flwr.common.logger import log
+from flwr.common.secure_aggregation.crypto.symmetric_encryption import (
+    bytes_to_public_key,
+    compute_hmac,
+    generate_shared_key,
+    public_key_to_bytes,
+)
+from flwr.proto.fab_pb2 import GetFabRequest  # pylint: disable=E0611
+from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
+    CreateNodeRequest,
+    DeleteNodeRequest,
+    PingRequest,
+    PullMessagesRequest,
+    PullTaskInsRequest,
+    PushMessagesRequest,
+    PushTaskResRequest,
+)
+from flwr.proto.run_pb2 import GetRunRequest  # pylint: disable=E0611
+
+_PUBLIC_KEY_HEADER = "public-key"
+_AUTH_TOKEN_HEADER = "auth-token"
+
+Request = Union[
+    CreateNodeRequest,
+    DeleteNodeRequest,
+    PullTaskInsRequest,
+    PushTaskResRequest,
+    GetRunRequest,
+    PingRequest,
+    GetFabRequest,
+    PullMessagesRequest,
+    PushMessagesRequest,
+]
+
+
+def _get_value_from_tuples(
+    key_string: str, tuples: Sequence[tuple[str, Union[str, bytes]]]
+) -> bytes:
+    value = next((value for key, value in tuples if key == key_string), "")
+    if isinstance(value, str):
+        return value.encode()
+
+    return value
+
+
+class _ClientCallDetails(
+    collections.namedtuple(
+        "_ClientCallDetails", ("method", "timeout", "metadata", "credentials")
+    ),
+    grpc.ClientCallDetails,  # type: ignore
+):
+    """Details for each client call.
+
+    The class will be passed on as the first argument in continuation function.
+    In our case, `AuthenticateClientInterceptor` adds new metadata to the construct.
+    """
+
+
+class AuthenticateClientInterceptor(grpc.UnaryUnaryClientInterceptor):  # type: ignore
+    """Client interceptor for client authentication."""
+
+    def __init__(
+        self,
+        private_key: ec.EllipticCurvePrivateKey,
+        public_key: ec.EllipticCurvePublicKey,
+    ):
+        self.private_key = private_key
+        self.public_key = public_key
+        self.shared_secret: Optional[bytes] = None
+        self.server_public_key: Optional[ec.EllipticCurvePublicKey] = None
+        self.encoded_public_key = base64.urlsafe_b64encode(
+            public_key_to_bytes(self.public_key)
+        )
+
+    def intercept_unary_unary(
+        self,
+        continuation: Callable[[Any, Any], Any],
+        client_call_details: grpc.ClientCallDetails,
+        request: Request,
+    ) -> grpc.Call:
+        """Flower client interceptor.
+
+        Intercept unary call from client and add necessary authentication header in the
+        RPC metadata.
+        """
+        metadata = []
+        postprocess = False
+        if client_call_details.metadata is not None:
+            metadata = list(client_call_details.metadata)
+
+        # Always add the public key header
+        metadata.append(
+            (
+                _PUBLIC_KEY_HEADER,
+                self.encoded_public_key,
+            )
+        )
+
+        if isinstance(request, CreateNodeRequest):
+            postprocess = True
+        elif isinstance(
+            request,
+            (
+                DeleteNodeRequest,
+                PullTaskInsRequest,
+                PushTaskResRequest,
+                GetRunRequest,
+                PingRequest,
+                GetFabRequest,
+                PullMessagesRequest,
+                PushMessagesRequest,
+            ),
+        ):
+            if self.shared_secret is None:
+                raise RuntimeError("Failure to compute hmac")
+
+            message_bytes = request.SerializeToString(deterministic=True)
+            metadata.append(
+                (
+                    _AUTH_TOKEN_HEADER,
+                    base64.urlsafe_b64encode(
+                        compute_hmac(self.shared_secret, message_bytes)
+                    ),
+                )
+            )
+
+        client_call_details = _ClientCallDetails(
+            client_call_details.method,
+            client_call_details.timeout,
+            metadata,
+            client_call_details.credentials,
+        )
+
+        response = continuation(client_call_details, request)
+        if postprocess:
+            server_public_key_bytes = base64.urlsafe_b64decode(
+                _get_value_from_tuples(_PUBLIC_KEY_HEADER, response.initial_metadata())
+            )
+
+            if server_public_key_bytes != b"":
+                self.server_public_key = bytes_to_public_key(server_public_key_bytes)
+            else:
+                log(WARNING, "Can't get server public key, SuperLink may be offline")
+
+            if self.server_public_key is not None:
+                self.shared_secret = generate_shared_key(
+                    self.private_key, self.server_public_key
+                )
+
+        return response

flwr/client/grpc_rere_client/connection.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Flower Labs GmbH. All Rights Reserved.
+# Copyright 2023 Flower Labs GmbH. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,32 +15,49 @@
 """Contextmanager for a gRPC request-response channel to the Flower server."""
 
 
+import random
+import threading
+from collections.abc import Iterator, Sequence
 from contextlib import contextmanager
 from copy import copy
 from logging import DEBUG, ERROR
 from pathlib import Path
-from typing import Callable, Dict, Iterator, Optional, Tuple, Union, cast
+from typing import Callable, Optional, Union, cast
 
+import grpc
+from cryptography.hazmat.primitives.asymmetric import ec
+
+from flwr.client.heartbeat import start_ping_loop
 from flwr.client.message_handler.message_handler import validate_out_message
-from flwr.client.message_handler.task_handler import get_task_ins, validate_task_ins
 from flwr.common import GRPC_MAX_MESSAGE_LENGTH
+from flwr.common.constant import (
+    PING_BASE_MULTIPLIER,
+    PING_CALL_TIMEOUT,
+    PING_DEFAULT_INTERVAL,
+    PING_RANDOM_RANGE,
+)
 from flwr.common.grpc import create_channel
-from flwr.common.logger import log, warn_experimental_feature
+from flwr.common.logger import log
 from flwr.common.message import Message, Metadata
 from flwr.common.retry_invoker import RetryInvoker
-from flwr.common.serde import message_from_taskins, message_to_taskres
+from flwr.common.serde import message_from_proto, message_to_proto, run_from_proto
+from flwr.common.typing import Fab, Run, RunNotRunningException
+from flwr.proto.fab_pb2 import GetFabRequest, GetFabResponse  # pylint: disable=E0611
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     DeleteNodeRequest,
-    PullTaskInsRequest,
-    PushTaskResRequest,
+    PingRequest,
+    PingResponse,
+    PullMessagesRequest,
+    PullMessagesResponse,
+    PushMessagesRequest,
 )
 from flwr.proto.fleet_pb2_grpc import FleetStub  # pylint: disable=E0611
 from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
-from flwr.proto.task_pb2 import TaskIns  # pylint: disable=E0611
+from flwr.proto.run_pb2 import GetRunRequest, GetRunResponse  # pylint: disable=E0611
 
-KEY_NODE = "node"
-KEY_METADATA = "in_message_metadata"
+from .client_interceptor import AuthenticateClientInterceptor
+from .grpc_adapter import GrpcAdapter
 
 
 def on_channel_state_change(channel_connectivity: str) -> None:
@@ -49,18 +66,24 @@ def on_channel_state_change(channel_connectivity: str) -> None:
 
 
 @contextmanager
-def grpc_request_response(
+def grpc_request_response(  # pylint: disable=R0913,R0914,R0915,R0917
     server_address: str,
     insecure: bool,
     retry_invoker: RetryInvoker,
     max_message_length: int = GRPC_MAX_MESSAGE_LENGTH,  # pylint: disable=W0613
     root_certificates: Optional[Union[bytes, str]] = None,
+    authentication_keys: Optional[
+        tuple[ec.EllipticCurvePrivateKey, ec.EllipticCurvePublicKey]
+    ] = None,
+    adapter_cls: Optional[Union[type[FleetStub], type[GrpcAdapter]]] = None,
 ) -> Iterator[
-    Tuple[
+    tuple[
         Callable[[], Optional[Message]],
         Callable[[Message], None],
+        Optional[Callable[[], Optional[int]]],
         Optional[Callable[[], None]],
-        Optional[Callable[[], None]],
+        Optional[Callable[[int], Run]],
+        Optional[Callable[[str, int], Fab]],
     ]
 ]:
     """Primitives for request/response-based interaction with a server.
@@ -87,6 +110,14 @@ def grpc_request_response(
         Path of the root certificate. If provided, a secure
         connection using the certificates will be established to an SSL-enabled
         Flower server. Bytes won't work for the REST API.
+    authentication_keys : Optional[Tuple[PrivateKey, PublicKey]] (default: None)
+        Tuple containing the elliptic curve private key and public key for
+        authentication from the cryptography library.
+        Source: https://cryptography.io/en/latest/hazmat/primitives/asymmetric/ec/
+        Used to establish an authenticated connection with the server.
+    adapter_cls: Optional[Union[type[FleetStub], type[GrpcAdapter]]] (default: None)
+        A GrpcStub Class that can be used to send messages. By default the FleetStub
+        will be used.
 
     Returns
     -------
@@ -94,113 +125,189 @@ def grpc_request_response(
     send : Callable
     create_node : Optional[Callable]
     delete_node : Optional[Callable]
+    get_run : Optional[Callable]
     """
-    warn_experimental_feature("`grpc-rere`")
-
     if isinstance(root_certificates, str):
         root_certificates = Path(root_certificates).read_bytes()
 
+    interceptors: Optional[Sequence[grpc.UnaryUnaryClientInterceptor]] = None
+    if authentication_keys is not None:
+        interceptors = AuthenticateClientInterceptor(
+            authentication_keys[0], authentication_keys[1]
+        )
+
     channel = create_channel(
         server_address=server_address,
         insecure=insecure,
         root_certificates=root_certificates,
         max_message_length=max_message_length,
+        interceptors=interceptors,
     )
     channel.subscribe(on_channel_state_change)
-    stub = FleetStub(channel)
 
-    # Necessary state to validate messages to be sent
-    state: Dict[str, Optional[Metadata]] = {KEY_METADATA: None}
-
-    # Enable create_node and delete_node to store node
-    node_store: Dict[str, Optional[Node]] = {KEY_NODE: None}
+    # Shared variables for inner functions
+    if adapter_cls is None:
+        adapter_cls = FleetStub
+    stub = adapter_cls(channel)
+    metadata: Optional[Metadata] = None
+    node: Optional[Node] = None
+    ping_thread: Optional[threading.Thread] = None
+    ping_stop_event = threading.Event()
+
+    def _should_giveup_fn(e: Exception) -> bool:
+        if e.code() == grpc.StatusCode.PERMISSION_DENIED:  # type: ignore
+            raise RunNotRunningException
+        if e.code() == grpc.StatusCode.UNAVAILABLE:  # type: ignore
+            return False
+        return True
+
+    # Restrict retries to cases where the status code is UNAVAILABLE
+    # If the status code is PERMISSION_DENIED, additionally raise RunNotRunningException
+    retry_invoker.should_giveup = _should_giveup_fn
 
     ###########################################################################
-    # receive/send functions
+    # ping/create_node/delete_node/receive/send/get_run functions
     ###########################################################################
 
-    def create_node() -> None:
+    def ping() -> None:
+        # Get Node
+        if node is None:
+            log(ERROR, "Node instance missing")
+            return
+
+        # Construct the ping request
+        req = PingRequest(node=node, ping_interval=PING_DEFAULT_INTERVAL)
+
+        # Call FleetAPI
+        res: PingResponse = stub.Ping(req, timeout=PING_CALL_TIMEOUT)
+
+        # Check if success
+        if not res.success:
+            raise RuntimeError("Ping failed unexpectedly.")
+
+        # Wait
+        rd = random.uniform(*PING_RANDOM_RANGE)
+        next_interval: float = PING_DEFAULT_INTERVAL - PING_CALL_TIMEOUT
+        next_interval *= PING_BASE_MULTIPLIER + rd
+        if not ping_stop_event.is_set():
+            ping_stop_event.wait(next_interval)
+
+    def create_node() -> Optional[int]:
         """Set create_node."""
-        create_node_request = CreateNodeRequest()
+        # Call FleetAPI
+        create_node_request = CreateNodeRequest(ping_interval=PING_DEFAULT_INTERVAL)
         create_node_response = retry_invoker.invoke(
             stub.CreateNode,
             request=create_node_request,
         )
-        node_store[KEY_NODE] = create_node_response.node
+
+        # Remember the node and the ping-loop thread
+        nonlocal node, ping_thread
+        node = cast(Node, create_node_response.node)
+        ping_thread = start_ping_loop(ping, ping_stop_event)
+        return node.node_id
 
     def delete_node() -> None:
         """Set delete_node."""
         # Get Node
-        if node_store[KEY_NODE] is None:
+        nonlocal node
+        if node is None:
             log(ERROR, "Node instance missing")
             return
-        node: Node = cast(Node, node_store[KEY_NODE])
 
+        # Stop the ping-loop thread
+        ping_stop_event.set()
+
+        # Call FleetAPI
         delete_node_request = DeleteNodeRequest(node=node)
         retry_invoker.invoke(stub.DeleteNode, request=delete_node_request)
 
-        del node_store[KEY_NODE]
+        # Cleanup
+        node = None
 
     def receive() -> Optional[Message]:
-        """Receive next task from server."""
+        """Receive next message from server."""
         # Get Node
-        if node_store[KEY_NODE] is None:
+        if node is None:
             log(ERROR, "Node instance missing")
             return None
-        node: Node = cast(Node, node_store[KEY_NODE])
 
-        # Request instructions (task) from server
-        request = PullTaskInsRequest(node=node)
-        response = retry_invoker.invoke(stub.PullTaskIns, request=request)
+        # Request instructions (message) from server
+        request = PullMessagesRequest(node=node)
+        response: PullMessagesResponse = retry_invoker.invoke(
+            stub.PullMessages, request=request
+        )
 
-        # Get the current TaskIns
-        task_ins: Optional[TaskIns] = get_task_ins(response)
+        # Get the current Messages
+        message_proto = (
+            None if len(response.messages_list) == 0 else response.messages_list[0]
+        )
 
-        # Discard the current TaskIns if not valid
-        if task_ins is not None and not (
-            task_ins.task.consumer.node_id == node.node_id
-            and validate_task_ins(task_ins)
+        # Discard the current message if not valid
+        if message_proto is not None and not (
+            message_proto.metadata.dst_node_id == node.node_id
         ):
-            task_ins = None
+            message_proto = None
 
         # Construct the Message
-        in_message = message_from_taskins(task_ins) if task_ins else None
+        in_message = message_from_proto(message_proto) if message_proto else None
 
         # Remember `metadata` of the in message
-        state[KEY_METADATA] = copy(in_message.metadata) if in_message else None
+        nonlocal metadata
+        metadata = copy(in_message.metadata) if in_message else None
 
         # Return the message if available
         return in_message
 
     def send(message: Message) -> None:
-        """Send task result back to server."""
+        """Send message reply to server."""
         # Get Node
-        if node_store[KEY_NODE] is None:
+        if node is None:
             log(ERROR, "Node instance missing")
             return
 
-        # Get incoming message
-        in_metadata = state[KEY_METADATA]
-        if in_metadata is None:
+        # Get the metadata of the incoming message
+        nonlocal metadata
+        if metadata is None:
             log(ERROR, "No current message")
             return
 
         # Validate out message
-        if not validate_out_message(message, in_metadata):
+        if not validate_out_message(message, metadata):
             log(ERROR, "Invalid out message")
             return
 
-        # Construct TaskRes
-        task_res = message_to_taskres(message)
+        # Serialize Message
+        message_proto = message_to_proto(message=message)
+        request = PushMessagesRequest(node=node, messages_list=[message_proto])
+        _ = retry_invoker.invoke(stub.PushMessages, request)
+
+        # Cleanup
+        metadata = None
+
+    def get_run(run_id: int) -> Run:
+        # Call FleetAPI
+        get_run_request = GetRunRequest(node=node, run_id=run_id)
+        get_run_response: GetRunResponse = retry_invoker.invoke(
+            stub.GetRun,
+            request=get_run_request,
+        )
+
+        # Return fab_id and fab_version
+        return run_from_proto(get_run_response.run)
 
-        # Serialize ProtoBuf to bytes
-        request = PushTaskResRequest(task_res_list=[task_res])
-        _ = retry_invoker.invoke(stub.PushTaskRes, request)
+    def get_fab(fab_hash: str, run_id: int) -> Fab:
+        # Call FleetAPI
+        get_fab_request = GetFabRequest(node=node, hash_str=fab_hash, run_id=run_id)
+        get_fab_response: GetFabResponse = retry_invoker.invoke(
+            stub.GetFab,
+            request=get_fab_request,
+        )
 
-        state[KEY_METADATA] = None
+        return Fab(get_fab_response.fab.hash_str, get_fab_response.fab.content)
 
     try:
         # Yield methods
-        yield (receive, send, create_node, delete_node)
+        yield (receive, send, create_node, delete_node, get_run, get_fab)
     except Exception as exc:  # pylint: disable=broad-except
         log(ERROR, exc)

flwr/client/grpc_rere_client/grpc_adapter.py

@@ -0,0 +1,167 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""GrpcAdapter implementation."""
+
+
+import sys
+from logging import DEBUG
+from typing import Any, TypeVar, cast
+
+import grpc
+from google.protobuf.message import Message as GrpcMessage
+
+from flwr.common import log
+from flwr.common.constant import (
+    GRPC_ADAPTER_METADATA_FLOWER_PACKAGE_NAME_KEY,
+    GRPC_ADAPTER_METADATA_FLOWER_PACKAGE_VERSION_KEY,
+    GRPC_ADAPTER_METADATA_FLOWER_VERSION_KEY,
+    GRPC_ADAPTER_METADATA_MESSAGE_MODULE_KEY,
+    GRPC_ADAPTER_METADATA_MESSAGE_QUALNAME_KEY,
+    GRPC_ADAPTER_METADATA_SHOULD_EXIT_KEY,
+)
+from flwr.common.version import package_name, package_version
+from flwr.proto.fab_pb2 import GetFabRequest, GetFabResponse  # pylint: disable=E0611
+from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
+    CreateNodeRequest,
+    CreateNodeResponse,
+    DeleteNodeRequest,
+    DeleteNodeResponse,
+    PingRequest,
+    PingResponse,
+    PullMessagesRequest,
+    PullMessagesResponse,
+    PullTaskInsRequest,
+    PullTaskInsResponse,
+    PushMessagesRequest,
+    PushMessagesResponse,
+    PushTaskResRequest,
+    PushTaskResResponse,
+)
+from flwr.proto.grpcadapter_pb2 import MessageContainer  # pylint: disable=E0611
+from flwr.proto.grpcadapter_pb2_grpc import GrpcAdapterStub
+from flwr.proto.run_pb2 import GetRunRequest, GetRunResponse  # pylint: disable=E0611
+
+T = TypeVar("T", bound=GrpcMessage)
+
+
+class GrpcAdapter:
+    """Adapter class to send and receive gRPC messages via the ``GrpcAdapterStub``.
+
+    This class utilizes the ``GrpcAdapterStub`` to send and receive gRPC messages
+    which are defined and used by the Fleet API, as defined in ``fleet.proto``.
+    """
+
+    def __init__(self, channel: grpc.Channel) -> None:
+        self.stub = GrpcAdapterStub(channel)
+
+    def _send_and_receive(
+        self, request: GrpcMessage, response_type: type[T], **kwargs: Any
+    ) -> T:
+        # Serialize request
+        req_cls = request.__class__
+        container_req = MessageContainer(
+            metadata={
+                GRPC_ADAPTER_METADATA_FLOWER_PACKAGE_NAME_KEY: package_name,
+                GRPC_ADAPTER_METADATA_FLOWER_PACKAGE_VERSION_KEY: package_version,
+                GRPC_ADAPTER_METADATA_FLOWER_VERSION_KEY: package_version,
+                GRPC_ADAPTER_METADATA_MESSAGE_MODULE_KEY: req_cls.__module__,
+                GRPC_ADAPTER_METADATA_MESSAGE_QUALNAME_KEY: req_cls.__qualname__,
+            },
+            grpc_message_name=req_cls.__qualname__,
+            grpc_message_content=request.SerializeToString(),
+        )
+
+        # Send via the stub
+        container_res = cast(
+            MessageContainer, self.stub.SendReceive(container_req, **kwargs)
+        )
+
+        # Handle control message
+        should_exit = (
+            container_res.metadata.get(GRPC_ADAPTER_METADATA_SHOULD_EXIT_KEY, "false")
+            == "true"
+        )
+        if should_exit:
+            log(
+                DEBUG,
+                'Received shutdown signal: exit flag is set to ``"true"``. Exiting...',
+            )
+            sys.exit(0)
+
+        # Check the grpc_message_name of the response
+        if container_res.grpc_message_name != response_type.__qualname__:
+            raise ValueError(
+                f"Invalid grpc_message_name. Expected {response_type.__qualname__}"
+                f", but got {container_res.grpc_message_name}."
+            )
+
+        # Deserialize response
+        response = response_type()
+        response.ParseFromString(container_res.grpc_message_content)
+        return response
+
+    def CreateNode(  # pylint: disable=C0103
+        self, request: CreateNodeRequest, **kwargs: Any
+    ) -> CreateNodeResponse:
+        """."""
+        return self._send_and_receive(request, CreateNodeResponse, **kwargs)
+
+    def DeleteNode(  # pylint: disable=C0103
+        self, request: DeleteNodeRequest, **kwargs: Any
+    ) -> DeleteNodeResponse:
+        """."""
+        return self._send_and_receive(request, DeleteNodeResponse, **kwargs)
+
+    def Ping(  # pylint: disable=C0103
+        self, request: PingRequest, **kwargs: Any
+    ) -> PingResponse:
+        """."""
+        return self._send_and_receive(request, PingResponse, **kwargs)
+
+    def PullTaskIns(  # pylint: disable=C0103
+        self, request: PullTaskInsRequest, **kwargs: Any
+    ) -> PullTaskInsResponse:
+        """."""
+        return self._send_and_receive(request, PullTaskInsResponse, **kwargs)
+
+    def PullMessages(  # pylint: disable=C0103
+        self, request: PullMessagesRequest, **kwargs: Any
+    ) -> PullMessagesResponse:
+        """."""
+        return self._send_and_receive(request, PullMessagesResponse, **kwargs)
+
+    def PushTaskRes(  # pylint: disable=C0103
+        self, request: PushTaskResRequest, **kwargs: Any
+    ) -> PushTaskResResponse:
+        """."""
+        return self._send_and_receive(request, PushTaskResResponse, **kwargs)
+
+    def PushMessages(  # pylint: disable=C0103
+        self, request: PushMessagesRequest, **kwargs: Any
+    ) -> PushMessagesResponse:
+        """."""
+        return self._send_and_receive(request, PushMessagesResponse, **kwargs)
+
+    def GetRun(  # pylint: disable=C0103
+        self, request: GetRunRequest, **kwargs: Any
+    ) -> GetRunResponse:
+        """."""
+        return self._send_and_receive(request, GetRunResponse, **kwargs)
+
+    def GetFab(  # pylint: disable=C0103
+        self, request: GetFabRequest, **kwargs: Any
+    ) -> GetFabResponse:
+        """."""
+        return self._send_and_receive(request, GetFabResponse, **kwargs)