flwr 1.20.0__py3-none-any.whl → 1.22.0__py3-none-any.whl

flwr/cli/pull.py

@@ -0,0 +1,100 @@
+# Copyright 2025 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower command line interface `pull` command."""
+
+
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+
+from flwr.cli.config_utils import (
+    exit_if_no_address,
+    load_and_validate,
+    process_loaded_project_config,
+    validate_federation_in_project_config,
+)
+from flwr.cli.constant import FEDERATION_CONFIG_HELP_MESSAGE
+from flwr.common.constant import FAB_CONFIG_FILE
+from flwr.proto.control_pb2 import (  # pylint: disable=E0611
+    PullArtifactsRequest,
+    PullArtifactsResponse,
+)
+from flwr.proto.control_pb2_grpc import ControlStub
+
+from .utils import flwr_cli_grpc_exc_handler, init_channel, try_obtain_cli_auth_plugin
+
+
+def pull(  # pylint: disable=R0914
+    run_id: Annotated[
+        int,
+        typer.Option(
+            "--run-id",
+            help="Run ID to pull artifacts from.",
+        ),
+    ],
+    app: Annotated[
+        Path,
+        typer.Argument(help="Path of the Flower App to run."),
+    ] = Path("."),
+    federation: Annotated[
+        Optional[str],
+        typer.Argument(help="Name of the federation."),
+    ] = None,
+    federation_config_overrides: Annotated[
+        Optional[list[str]],
+        typer.Option(
+            "--federation-config",
+            help=FEDERATION_CONFIG_HELP_MESSAGE,
+        ),
+    ] = None,
+) -> None:
+    """Pull artifacts from a Flower run."""
+    typer.secho("Loading project configuration... ", fg=typer.colors.BLUE)
+
+    pyproject_path = app / FAB_CONFIG_FILE if app else None
+    config, errors, warnings = load_and_validate(path=pyproject_path)
+    config = process_loaded_project_config(config, errors, warnings)
+    federation, federation_config = validate_federation_in_project_config(
+        federation, config, federation_config_overrides
+    )
+    exit_if_no_address(federation_config, "pull")
+    channel = None
+    try:
+
+        auth_plugin = try_obtain_cli_auth_plugin(app, federation, federation_config)
+        channel = init_channel(app, federation_config, auth_plugin)
+        stub = ControlStub(channel)
+        with flwr_cli_grpc_exc_handler():
+            res: PullArtifactsResponse = stub.PullArtifacts(
+                PullArtifactsRequest(run_id=run_id)
+            )
+
+        if not res.url:
+            typer.secho(
+                f"❌ A download URL for artifacts from run {run_id} couldn't be "
+                "obtained.",
+                fg=typer.colors.RED,
+                bold=True,
+            )
+            raise typer.Exit(code=1)
+
+        typer.secho(
+            f"✅ Artifacts for run {run_id} can be downloaded from: {res.url}",
+            fg=typer.colors.GREEN,
+        )
+    finally:
+        if channel:
+            channel.close()

flwr/cli/run/run.py

@@ -30,7 +30,7 @@ from flwr.cli.config_utils import (
     process_loaded_project_config,
     validate_federation_in_project_config,
 )
-from flwr.cli.constant import FEDERATION_CONFIG_HELP_MESSAGE
+from flwr.cli.constant import FEDERATION_CONFIG_HELP_MESSAGE, RUN_CONFIG_HELP_MESSAGE
 from flwr.common.config import (
     flatten_dict,
     get_metadata_from_config,
@@ -41,8 +41,8 @@ from flwr.common.constant import CliOutputFormat
 from flwr.common.logger import print_json_error, redirect_output, restore_output
 from flwr.common.serde import config_record_to_proto, fab_to_proto, user_config_to_proto
 from flwr.common.typing import Fab
-from flwr.proto.exec_pb2 import StartRunRequest  # pylint: disable=E0611
-from flwr.proto.exec_pb2_grpc import ExecStub
+from flwr.proto.control_pb2 import StartRunRequest  # pylint: disable=E0611
+from flwr.proto.control_pb2_grpc import ControlStub
 
 from ..log import start_stream
 from ..utils import flwr_cli_grpc_exc_handler, init_channel, try_obtain_cli_auth_plugin
@@ -65,11 +65,7 @@ def run(
         typer.Option(
             "--run-config",
             "-c",
-            help="Override run configuration values in the format:\n\n"
-            "`--run-config 'key1=value1 key2=value2' --run-config 'key3=value3'`\n\n"
-            "Values can be of any type supported in TOML, such as bool, int, "
-            "float, or string. Ensure that the keys (`key1`, `key2`, `key3` "
-            "in this example) exist in `pyproject.toml` for proper overriding.",
+            help=RUN_CONFIG_HELP_MESSAGE,
         ),
     ] = None,
     federation_config_overrides: Annotated[
@@ -112,7 +108,7 @@ def run(
         )
 
         if "address" in federation_config:
-            _run_with_exec_api(
+            _run_with_control_api(
                 app,
                 federation,
                 federation_config,
@@ -121,7 +117,7 @@ def run(
                 output_format,
             )
         else:
-            _run_without_exec_api(
+            _run_without_control_api(
                 app, federation_config, run_config_overrides, federation
             )
     except (typer.Exit, Exception) as err:  # pylint: disable=broad-except
@@ -142,7 +138,7 @@ def run(
 
 
 # pylint: disable-next=R0913, R0914, R0917
-def _run_with_exec_api(
+def _run_with_control_api(
     app: Path,
     federation: str,
     federation_config: dict[str, Any],
@@ -154,7 +150,7 @@ def _run_with_exec_api(
     try:
         auth_plugin = try_obtain_cli_auth_plugin(app, federation, federation_config)
         channel = init_channel(app, federation_config, auth_plugin)
-        stub = ExecStub(channel)
+        stub = ControlStub(channel)
 
         fab_bytes, fab_hash, config = build_fab(app)
         fab_id, fab_version = get_metadata_from_config(config)
@@ -203,7 +199,7 @@ def _run_with_exec_api(
             channel.close()
 
 
-def _run_without_exec_api(
+def _run_without_control_api(
     app: Optional[Path],
     federation_config: dict[str, Any],
     config_overrides: Optional[list[str]],

flwr/cli/stop.py

@@ -32,8 +32,11 @@ from flwr.cli.config_utils import (
 from flwr.cli.constant import FEDERATION_CONFIG_HELP_MESSAGE
 from flwr.common.constant import FAB_CONFIG_FILE, CliOutputFormat
 from flwr.common.logger import print_json_error, redirect_output, restore_output
-from flwr.proto.exec_pb2 import StopRunRequest, StopRunResponse  # pylint: disable=E0611
-from flwr.proto.exec_pb2_grpc import ExecStub
+from flwr.proto.control_pb2 import (  # pylint: disable=E0611
+    StopRunRequest,
+    StopRunResponse,
+)
+from flwr.proto.control_pb2_grpc import ControlStub
 
 from .utils import flwr_cli_grpc_exc_handler, init_channel, try_obtain_cli_auth_plugin
 
@@ -88,7 +91,7 @@ def stop(  # pylint: disable=R0914
         try:
             auth_plugin = try_obtain_cli_auth_plugin(app, federation, federation_config)
             channel = init_channel(app, federation_config, auth_plugin)
-            stub = ExecStub(channel)  # pylint: disable=unused-variable # noqa: F841
+            stub = ControlStub(channel)  # pylint: disable=unused-variable # noqa: F841
 
             typer.secho(f"✋ Stopping run ID {run_id}...", fg=typer.colors.GREEN)
             _stop_run(stub=stub, run_id=run_id, output_format=output_format)
@@ -120,7 +123,7 @@ def stop(  # pylint: disable=R0914
         captured_output.close()
 
 
-def _stop_run(stub: ExecStub, run_id: int, output_format: str) -> None:
+def _stop_run(stub: ControlStub, run_id: int, output_format: str) -> None:
     """Stop a run."""
     with flwr_cli_grpc_exc_handler():
         response: StopRunResponse = stub.StopRun(request=StopRunRequest(run_id=run_id))

flwr/cli/utils.py

@@ -32,6 +32,9 @@ from flwr.common.constant import (
     AUTH_TYPE_JSON_KEY,
     CREDENTIALS_DIR,
     FLWR_DIR,
+    NO_ARTIFACT_PROVIDER_MESSAGE,
+    NO_USER_AUTH_MESSAGE,
+    PULL_UNFINISHED_RUN_MESSAGE,
     RUN_ID_NOT_FOUND_MESSAGE,
 )
 from flwr.common.grpc import (
@@ -259,7 +262,7 @@ def try_obtain_cli_auth_plugin(
 def init_channel(
     app: Path, federation_config: dict[str, Any], auth_plugin: Optional[CliAuthPlugin]
 ) -> grpc.Channel:
-    """Initialize gRPC channel to the Exec API."""
+    """Initialize gRPC channel to the Control API."""
     insecure, root_certificates_bytes = validate_certificate_in_federation_config(
         app, federation_config
     )
@@ -312,11 +315,27 @@ def flwr_cli_grpc_exc_handler() -> Iterator[None]:
             )
             raise typer.Exit(code=1) from None
         if e.code() == grpc.StatusCode.UNIMPLEMENTED:
-            typer.secho(
-                "❌ User authentication is not enabled on this SuperLink.",
-                fg=typer.colors.RED,
-                bold=True,
-            )
+            if e.details() == NO_USER_AUTH_MESSAGE:  # pylint: disable=E1101
+                typer.secho(
+                    "❌ User authentication is not enabled on this SuperLink.",
+                    fg=typer.colors.RED,
+                    bold=True,
+                )
+            elif e.details() == NO_ARTIFACT_PROVIDER_MESSAGE:  # pylint: disable=E1101
+                typer.secho(
+                    "❌ The SuperLink does not support `flwr pull` command.",
+                    fg=typer.colors.RED,
+                    bold=True,
+                )
+            else:
+                typer.secho(
+                    "❌ The SuperLink cannot process this request. Please verify that "
+                    "you set the address to its Control API endpoint correctly in your "
+                    "`pyproject.toml`, and ensure that the Flower versions used by "
+                    "the CLI and SuperLink are compatible.",
+                    fg=typer.colors.RED,
+                    bold=True,
+                )
             raise typer.Exit(code=1) from None
         if e.code() == grpc.StatusCode.PERMISSION_DENIED:
             typer.secho(
@@ -324,7 +343,7 @@ def flwr_cli_grpc_exc_handler() -> Iterator[None]:
                 fg=typer.colors.RED,
                 bold=True,
             )
-            # pylint: disable=E1101
+            # pylint: disable-next=E1101
             typer.secho(e.details(), fg=typer.colors.RED, bold=True)
             raise typer.Exit(code=1) from None
         if e.code() == grpc.StatusCode.UNAVAILABLE:
@@ -337,7 +356,7 @@ def flwr_cli_grpc_exc_handler() -> Iterator[None]:
             raise typer.Exit(code=1) from None
         if (
             e.code() == grpc.StatusCode.NOT_FOUND
-            and e.details() == RUN_ID_NOT_FOUND_MESSAGE
+            and e.details() == RUN_ID_NOT_FOUND_MESSAGE  # pylint: disable=E1101
         ):
             typer.secho(
                 "❌ Run ID not found.",
@@ -345,4 +364,13 @@ def flwr_cli_grpc_exc_handler() -> Iterator[None]:
                 bold=True,
             )
             raise typer.Exit(code=1) from None
+        if e.code() == grpc.StatusCode.FAILED_PRECONDITION:
+            if e.details() == PULL_UNFINISHED_RUN_MESSAGE:  # pylint: disable=E1101
+                typer.secho(
+                    "❌ Run is not finished yet. Artifacts can only be pulled after "
+                    "the run is finished. You can check the run status with `flwr ls`.",
+                    fg=typer.colors.RED,
+                    bold=True,
+                )
+                raise typer.Exit(code=1) from None
         raise

flwr/client/grpc_rere_client/connection.py

@@ -40,7 +40,7 @@ from flwr.common.secure_aggregation.crypto.symmetric_encryption import (
     generate_key_pairs,
 )
 from flwr.common.serde import message_from_proto, message_to_proto, run_from_proto
-from flwr.common.typing import Fab, Run, RunNotRunningException
+from flwr.common.typing import Fab, Run
 from flwr.proto.fab_pb2 import GetFabRequest, GetFabResponse  # pylint: disable=E0611
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
@@ -157,17 +157,6 @@ def grpc_request_response(  # pylint: disable=R0913,R0914,R0915,R0917
     stub = adapter_cls(channel)
     node: Optional[Node] = None
 
-    def _should_giveup_fn(e: Exception) -> bool:
-        if e.code() == grpc.StatusCode.PERMISSION_DENIED:  # type: ignore
-            raise RunNotRunningException
-        if e.code() == grpc.StatusCode.UNAVAILABLE:  # type: ignore
-            return False
-        return True
-
-    # Restrict retries to cases where the status code is UNAVAILABLE
-    # If the status code is PERMISSION_DENIED, additionally raise RunNotRunningException
-    retry_invoker.should_giveup = _should_giveup_fn
-
     # Wrap stub
     _wrap_stub(stub, retry_invoker)
     ###########################################################################

flwr/client/rest_client/connection.py

@@ -176,6 +176,9 @@ def http_request_response(  # pylint: disable=R0913,R0914,R0915,R0917
     # Shared variables for inner functions
     node: Optional[Node] = None
 
+    # Remove should_giveup from RetryInvoker as REST does not support gRPC status codes
+    retry_invoker.should_giveup = None
+
     ###########################################################################
     # heartbeat/create_node/delete_node/receive/send/get_run functions
     ###########################################################################

flwr/clientapp/__init__.py

@@ -13,3 +13,13 @@
 # limitations under the License.
 # ==============================================================================
 """Public Flower ClientApp APIs."""
+
+
+from flwr.client.client_app import ClientApp
+
+from . import mod
+
+__all__ = [
+    "ClientApp",
+    "mod",
+]

flwr/clientapp/mod/__init__.py

@@ -0,0 +1,29 @@
+# Copyright 2025 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower Built-in Mods."""
+
+
+from flwr.client.mod.comms_mods import arrays_size_mod, message_size_mod
+
+from .centraldp_mods import adaptiveclipping_mod, fixedclipping_mod
+from .localdp_mod import LocalDpMod
+
+__all__ = [
+    "LocalDpMod",
+    "adaptiveclipping_mod",
+    "arrays_size_mod",
+    "fixedclipping_mod",
+    "message_size_mod",
+]

flwr/clientapp/mod/centraldp_mods.py

@@ -0,0 +1,248 @@
+# Copyright 2025 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Clipping modifiers for central DP with client-side clipping."""
+
+
+from collections import OrderedDict
+from logging import ERROR, INFO
+from typing import cast
+
+from flwr.app import Error
+from flwr.clientapp.typing import ClientAppCallable
+from flwr.common import (
+    Array,
+    ArrayRecord,
+    ConfigRecord,
+    Context,
+    Message,
+    MetricRecord,
+    log,
+)
+from flwr.common.constant import ErrorCode
+from flwr.common.differential_privacy import (
+    compute_adaptive_clip_model_update,
+    compute_clip_model_update,
+)
+from flwr.common.differential_privacy_constants import KEY_CLIPPING_NORM, KEY_NORM_BIT
+
+
+# pylint: disable=too-many-return-statements
+def fixedclipping_mod(
+    msg: Message, ctxt: Context, call_next: ClientAppCallable
+) -> Message:
+    """Client-side fixed clipping modifier.
+
+    This mod needs to be used with the `DifferentialPrivacyClientSideFixedClipping`
+    server-side strategy wrapper.
+
+    The wrapper sends the clipping_norm value to the client.
+
+    This mod clips the client model updates before sending them to the server.
+
+    It operates on messages of type `MessageType.TRAIN`.
+
+    Notes
+    -----
+    Consider the order of mods when using multiple.
+
+    Typically, fixedclipping_mod should be the last to operate on params.
+    """
+    if len(msg.content.array_records) != 1:
+        return _handle_multi_record_err("fixedclipping_mod", msg, ArrayRecord)
+    if len(msg.content.config_records) != 1:
+        return _handle_multi_record_err("fixedclipping_mod", msg, ConfigRecord)
+
+    # Get keys in the single ConfigRecord
+    keys_in_config = set(next(iter(msg.content.config_records.values())).keys())
+    if KEY_CLIPPING_NORM not in keys_in_config:
+        return _handle_no_key_err("fixedclipping_mod", msg)
+    # Record array record communicated to client and clipping norm
+    original_array_record = next(iter(msg.content.array_records.values()))
+    clipping_norm = cast(
+        float, next(iter(msg.content.config_records.values()))[KEY_CLIPPING_NORM]
+    )
+
+    # Call inner app
+    out_msg = call_next(msg, ctxt)
+
+    # Check if the msg has error
+    if out_msg.has_error():
+        return out_msg
+
+    # Ensure reply has a single ArrayRecord
+    if len(out_msg.content.array_records) != 1:
+        return _handle_multi_record_err("fixedclipping_mod", out_msg, ArrayRecord)
+
+    new_array_record_key, client_to_server_arrecord = next(
+        iter(out_msg.content.array_records.items())
+    )
+    # Ensure keys in returned ArrayRecord match those in the one sent from server
+    if list(original_array_record.keys()) != list(client_to_server_arrecord.keys()):
+        return _handle_array_key_mismatch_err("fixedclipping_mod", out_msg)
+
+    client_to_server_ndarrays = client_to_server_arrecord.to_numpy_ndarrays()
+    # Clip the client update
+    compute_clip_model_update(
+        param1=client_to_server_ndarrays,
+        param2=original_array_record.to_numpy_ndarrays(),
+        clipping_norm=clipping_norm,
+    )
+
+    log(
+        INFO, "fixedclipping_mod: parameters are clipped by value: %.4f.", clipping_norm
+    )
+    # Replace outgoing ArrayRecord's Array while preserving their keys
+    out_msg.content.array_records[new_array_record_key] = ArrayRecord(
+        OrderedDict(
+            {
+                k: Array(v)
+                for k, v in zip(
+                    client_to_server_arrecord.keys(), client_to_server_ndarrays
+                )
+            }
+        )
+    )
+    return out_msg
+
+
+def adaptiveclipping_mod(
+    msg: Message, ctxt: Context, call_next: ClientAppCallable
+) -> Message:
+    """Client-side adaptive clipping modifier.
+
+    This mod needs to be used with the DifferentialPrivacyClientSideAdaptiveClipping
+    server-side strategy wrapper.
+
+    The wrapper sends the clipping_norm value to the client.
+
+    This mod clips the client model updates before sending them to the server.
+
+    It also sends KEY_NORM_BIT to the server for computing the new clipping value.
+
+    It operates on messages of type `MessageType.TRAIN`.
+
+    Notes
+    -----
+    Consider the order of mods when using multiple.
+
+    Typically, adaptiveclipping_mod should be the last to operate on params.
+    """
+    if len(msg.content.array_records) != 1:
+        return _handle_multi_record_err("adaptiveclipping_mod", msg, ArrayRecord)
+    if len(msg.content.config_records) != 1:
+        return _handle_multi_record_err("adaptiveclipping_mod", msg, ConfigRecord)
+
+    # Get keys in the single ConfigRecord
+    keys_in_config = set(next(iter(msg.content.config_records.values())).keys())
+    if KEY_CLIPPING_NORM not in keys_in_config:
+        return _handle_no_key_err("adaptiveclipping_mod", msg)
+
+    # Record array record communicated to client and clipping norm
+    original_array_record = next(iter(msg.content.array_records.values()))
+    clipping_norm = cast(
+        float, next(iter(msg.content.config_records.values()))[KEY_CLIPPING_NORM]
+    )
+
+    # Call inner app
+    out_msg = call_next(msg, ctxt)
+
+    # Check if the msg has error
+    if out_msg.has_error():
+        return out_msg
+
+    # Ensure reply has a single ArrayRecord
+    if len(out_msg.content.array_records) != 1:
+        return _handle_multi_record_err("adaptiveclipping_mod", out_msg, ArrayRecord)
+
+    # Ensure reply has a single MetricRecord
+    if len(out_msg.content.metric_records) != 1:
+        return _handle_multi_record_err("adaptiveclipping_mod", out_msg, MetricRecord)
+
+    new_array_record_key, client_to_server_arrecord = next(
+        iter(out_msg.content.array_records.items())
+    )
+
+    # Ensure keys in returned ArrayRecord match those in the one sent from server
+    if list(original_array_record.keys()) != list(client_to_server_arrecord.keys()):
+        return _handle_array_key_mismatch_err("adaptiveclipping_mod", out_msg)
+
+    client_to_server_ndarrays = client_to_server_arrecord.to_numpy_ndarrays()
+    # Clip the client update
+    norm_bit = compute_adaptive_clip_model_update(
+        client_to_server_ndarrays,
+        original_array_record.to_numpy_ndarrays(),
+        clipping_norm,
+    )
+    log(
+        INFO,
+        "adaptiveclipping_mod: ndarrays are clipped by value: %.4f.",
+        clipping_norm,
+    )
+    # Replace outgoing ArrayRecord's Array while preserving their keys
+    out_msg.content.array_records[new_array_record_key] = ArrayRecord(
+        OrderedDict(
+            {
+                k: Array(v)
+                for k, v in zip(
+                    client_to_server_arrecord.keys(), client_to_server_ndarrays
+                )
+            }
+        )
+    )
+    # Add to the MetricRecords the norm bit (recall reply messages only contain
+    # one MetricRecord)
+    metric_record_key = list(out_msg.content.metric_records.keys())[0]
+    # We cast it to `int` because MetricRecord does not support `bool` values
+    out_msg.content.metric_records[metric_record_key][KEY_NORM_BIT] = int(norm_bit)
+    return out_msg
+
+
+def _handle_err(msg: Message, reason: str) -> Message:
+    """Log and return error message."""
+    log(ERROR, reason)
+    return Message(
+        Error(code=ErrorCode.MOD_FAILED_PRECONDITION, reason=reason),
+        reply_to=msg,
+    )
+
+
+def _handle_multi_record_err(mod_name: str, msg: Message, record_type: type) -> Message:
+    """Log and return multi-record error."""
+    cnt = sum(isinstance(_, record_type) for _ in msg.content.values())
+    return _handle_err(
+        msg,
+        f"{mod_name} expects exactly one {record_type.__name__}, "
+        f"but found {cnt} {record_type.__name__}(s).",
+    )
+
+
+def _handle_no_key_err(mod_name: str, msg: Message) -> Message:
+    """Log and return no-key error."""
+    return _handle_err(
+        msg,
+        f"{mod_name} requires the key '{KEY_CLIPPING_NORM}' to be present in the "
+        "ConfigRecord, but it was not found. "
+        "Please ensure the `DifferentialPrivacyClientSideFixedClipping` wrapper "
+        "is used in the ServerApp.",
+    )
+
+
+def _handle_array_key_mismatch_err(mod_name: str, msg: Message) -> Message:
+    """Create array-key-mismatch error reasons."""
+    return _handle_err(
+        msg,
+        f"{mod_name} expects the keys in the ArrayRecord of the reply message to match "
+        "those from the ArrayRecord that the ClientApp received, but they do not.",
+    )