flwr-nightly 1.11.0.dev20240812__py3-none-any.whl → 1.11.0.dev20240815__py3-none-any.whl

flwr/cli/run/run.py

@@ -14,6 +14,7 @@
 # ==============================================================================
 """Flower command line interface `run` command."""
 
+import hashlib
 import subprocess
 import sys
 from logging import DEBUG
@@ -28,7 +29,8 @@ from flwr.cli.config_utils import load_and_validate
 from flwr.common.config import flatten_dict, parse_config_args
 from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel
 from flwr.common.logger import log
-from flwr.common.serde import user_config_to_proto
+from flwr.common.serde import fab_to_proto, user_config_to_proto
+from flwr.common.typing import Fab
 from flwr.proto.exec_pb2 import StartRunRequest  # pylint: disable=E0611
 from flwr.proto.exec_pb2_grpc import ExecStub
 
@@ -163,9 +165,11 @@ def _run_with_superexec(
     stub = ExecStub(channel)
 
     fab_path = Path(build(app))
+    content = fab_path.read_bytes()
+    fab = Fab(hashlib.sha256(content).hexdigest(), content)
 
     req = StartRunRequest(
-        fab_file=fab_path.read_bytes(),
+        fab=fab_to_proto(fab),
         override_config=user_config_to_proto(
             parse_config_args(config_overrides, separator=",")
         ),

flwr/client/app.py

@@ -42,7 +42,7 @@ from flwr.common.constant import (
 from flwr.common.logger import log, warn_deprecated_feature
 from flwr.common.message import Error
 from flwr.common.retry_invoker import RetryInvoker, RetryState, exponential
-from flwr.common.typing import Run, UserConfig
+from flwr.common.typing import Fab, Run, UserConfig
 
 from .grpc_adapter_client.connection import grpc_adapter
 from .grpc_client.connection import grpc_connection
@@ -333,7 +333,7 @@ def _start_client_internal(
             root_certificates,
             authentication_keys,
         ) as conn:
-            receive, send, create_node, delete_node, get_run = conn
+            receive, send, create_node, delete_node, get_run, _ = conn
 
             # Register node when connecting the first time
             if node_state is None:
@@ -398,7 +398,7 @@ def _start_client_internal(
                             runs[run_id] = get_run(run_id)
                         # If get_run is None, i.e., in grpc-bidi mode
                         else:
-                            runs[run_id] = Run(run_id, "", "", {})
+                            runs[run_id] = Run(run_id, "", "", "", {})
 
                     # Register context for this run
                     node_state.register_context(
@@ -606,6 +606,7 @@ def _init_connection(transport: Optional[str], server_address: str) -> Tuple[
                 Optional[Callable[[], Optional[int]]],
                 Optional[Callable[[], None]],
                 Optional[Callable[[int], Run]],
+                Optional[Callable[[str], Fab]],
             ]
         ],
     ],

flwr/client/grpc_adapter_client/connection.py

@@ -27,7 +27,7 @@ from flwr.common import GRPC_MAX_MESSAGE_LENGTH
 from flwr.common.logger import log
 from flwr.common.message import Message
 from flwr.common.retry_invoker import RetryInvoker
-from flwr.common.typing import Run
+from flwr.common.typing import Fab, Run
 
 
 @contextmanager
@@ -47,6 +47,7 @@ def grpc_adapter(  # pylint: disable=R0913
         Optional[Callable[[], Optional[int]]],
         Optional[Callable[[], None]],
         Optional[Callable[[int], Run]],
+        Optional[Callable[[str], Fab]],
     ]
 ]:
     """Primitives for request/response-based interaction with a server via GrpcAdapter.
@@ -80,6 +81,7 @@ def grpc_adapter(  # pylint: disable=R0913
     create_node : Optional[Callable]
     delete_node : Optional[Callable]
     get_run : Optional[Callable]
+    get_fab : Optional[Callable]
     """
     if authentication_keys is not None:
         log(ERROR, "Client authentication is not supported for this transport type.")

flwr/client/grpc_client/connection.py

@@ -38,7 +38,7 @@ from flwr.common.constant import MessageType, MessageTypeLegacy
 from flwr.common.grpc import create_channel
 from flwr.common.logger import log
 from flwr.common.retry_invoker import RetryInvoker
-from flwr.common.typing import Run
+from flwr.common.typing import Fab, Run
 from flwr.proto.transport_pb2 import (  # pylint: disable=E0611
     ClientMessage,
     Reason,
@@ -75,6 +75,7 @@ def grpc_connection(  # pylint: disable=R0913, R0915
         Optional[Callable[[], Optional[int]]],
         Optional[Callable[[], None]],
         Optional[Callable[[int], Run]],
+        Optional[Callable[[str], Fab]],
     ]
 ]:
     """Establish a gRPC connection to a gRPC server.
@@ -235,7 +236,7 @@ def grpc_connection(  # pylint: disable=R0913, R0915
 
     try:
         # Yield methods
-        yield (receive, send, None, None, None)
+        yield (receive, send, None, None, None, None)
     finally:
         # Make sure to have a final
         channel.close()

flwr/client/grpc_rere_client/connection.py

@@ -45,7 +45,7 @@ from flwr.common.serde import (
     message_to_taskres,
     user_config_from_proto,
 )
-from flwr.common.typing import Run
+from flwr.common.typing import Fab, Run
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     DeleteNodeRequest,
@@ -86,6 +86,7 @@ def grpc_request_response(  # pylint: disable=R0913, R0914, R0915
         Optional[Callable[[], Optional[int]]],
         Optional[Callable[[], None]],
         Optional[Callable[[int], Run]],
+        Optional[Callable[[str], Fab]],
     ]
 ]:
     """Primitives for request/response-based interaction with a server.
@@ -285,11 +286,16 @@ def grpc_request_response(  # pylint: disable=R0913, R0914, R0915
             run_id,
             get_run_response.run.fab_id,
             get_run_response.run.fab_version,
+            get_run_response.run.fab_hash,
             user_config_from_proto(get_run_response.run.override_config),
         )
 
+    def get_fab(fab_hash: str) -> Fab:
+        # Call FleetAPI
+        raise NotImplementedError
+
     try:
         # Yield methods
-        yield (receive, send, create_node, delete_node, get_run)
+        yield (receive, send, create_node, delete_node, get_run, get_fab)
     except Exception as exc:  # pylint: disable=broad-except
         log(ERROR, exc)

flwr/client/process/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower AppIO service."""

flwr/client/process/clientappio_servicer.py

@@ -0,0 +1,145 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""ClientAppIo API servicer."""
+
+
+from dataclasses import dataclass
+from logging import DEBUG, ERROR
+from typing import Optional
+
+import grpc
+
+from flwr.common import Context, Message, typing
+from flwr.common.logger import log
+from flwr.common.serde import (
+    clientappstatus_to_proto,
+    context_from_proto,
+    context_to_proto,
+    message_from_proto,
+    message_to_proto,
+    run_to_proto,
+)
+from flwr.common.typing import Run
+
+# pylint: disable=E0611
+from flwr.proto import clientappio_pb2_grpc
+from flwr.proto.clientappio_pb2 import (  # pylint: disable=E0401
+    PullClientAppInputsRequest,
+    PullClientAppInputsResponse,
+    PushClientAppOutputsRequest,
+    PushClientAppOutputsResponse,
+)
+
+
+@dataclass
+class ClientAppIoInputs:
+    """Specify the inputs to the ClientApp."""
+
+    message: Message
+    context: Context
+    run: Run
+    token: int
+
+
+@dataclass
+class ClientAppIoOutputs:
+    """Specify the outputs from the ClientApp."""
+
+    message: Message
+    context: Context
+
+
+# pylint: disable=C0103,W0613,W0201
+class ClientAppIoServicer(clientappio_pb2_grpc.ClientAppIoServicer):
+    """ClientAppIo API servicer."""
+
+    def __init__(self) -> None:
+        self.clientapp_input: Optional[ClientAppIoInputs] = None
+        self.clientapp_output: Optional[ClientAppIoOutputs] = None
+
+    def PullClientAppInputs(
+        self, request: PullClientAppInputsRequest, context: grpc.ServicerContext
+    ) -> PullClientAppInputsResponse:
+        """Pull Message, Context, and Run."""
+        log(DEBUG, "ClientAppIo.PullClientAppInputs")
+        if self.clientapp_input is None:
+            raise ValueError(
+                "ClientAppIoInputs not set before calling `PullClientAppInputs`."
+            )
+        if request.token != self.clientapp_input.token:
+            context.abort(
+                grpc.StatusCode.INVALID_ARGUMENT,
+                "Mismatch between ClientApp and SuperNode token",
+            )
+        return PullClientAppInputsResponse(
+            message=message_to_proto(self.clientapp_input.message),
+            context=context_to_proto(self.clientapp_input.context),
+            run=run_to_proto(self.clientapp_input.run),
+        )
+
+    def PushClientAppOutputs(
+        self, request: PushClientAppOutputsRequest, context: grpc.ServicerContext
+    ) -> PushClientAppOutputsResponse:
+        """Push Message and Context."""
+        log(DEBUG, "ClientAppIo.PushClientAppOutputs")
+        if self.clientapp_output is None:
+            raise ValueError(
+                "ClientAppIoOutputs not set before calling `PushClientAppOutputs`."
+            )
+        if self.clientapp_input is None:
+            raise ValueError(
+                "ClientAppIoInputs not set before calling `PushClientAppOutputs`."
+            )
+        if request.token != self.clientapp_input.token:
+            context.abort(
+                grpc.StatusCode.INVALID_ARGUMENT,
+                "Mismatch between ClientApp and SuperNode token",
+            )
+        try:
+            # Update Message and Context
+            self.clientapp_output.message = message_from_proto(request.message)
+            self.clientapp_output.context = context_from_proto(request.context)
+            # Set status
+            code = typing.ClientAppOutputCode.SUCCESS
+            status = typing.ClientAppOutputStatus(code=code, message="Success")
+            proto_status = clientappstatus_to_proto(status=status)
+            return PushClientAppOutputsResponse(status=proto_status)
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            log(ERROR, "ClientApp failed to push message to SuperNode, %s", e)
+            code = typing.ClientAppOutputCode.UNKNOWN_ERROR
+            status = typing.ClientAppOutputStatus(code=code, message="Push failed")
+            proto_status = clientappstatus_to_proto(status=status)
+            return PushClientAppOutputsResponse(status=proto_status)
+
+    def set_inputs(self, clientapp_input: ClientAppIoInputs) -> None:
+        """Set ClientApp inputs."""
+        log(DEBUG, "ClientAppIo.SetInputs")
+        if self.clientapp_input is not None or self.clientapp_output is not None:
+            raise ValueError(
+                "ClientAppIoInputs and ClientAppIoOutputs must not be set before "
+                "calling `set_inputs`."
+            )
+        self.clientapp_input = clientapp_input
+
+    def get_outputs(self) -> ClientAppIoOutputs:
+        """Get ClientApp outputs."""
+        log(DEBUG, "ClientAppIo.GetOutputs")
+        if self.clientapp_output is None:
+            raise ValueError("ClientAppIoOutputs not set before calling `get_outputs`.")
+        # Set outputs to a local variable and clear self.clientapp_output
+        output: ClientAppIoOutputs = self.clientapp_output
+        self.clientapp_input = None
+        self.clientapp_output = None
+        return output

flwr/client/rest_client/connection.py

@@ -45,7 +45,7 @@ from flwr.common.serde import (
     message_to_taskres,
     user_config_from_proto,
 )
-from flwr.common.typing import Run
+from flwr.common.typing import Fab, Run
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     CreateNodeResponse,
@@ -97,6 +97,7 @@ def http_request_response(  # pylint: disable=,R0913, R0914, R0915
         Optional[Callable[[], Optional[int]]],
         Optional[Callable[[], None]],
         Optional[Callable[[int], Run]],
+        Optional[Callable[[str], Fab]],
     ]
 ]:
     """Primitives for request/response-based interaction with a server.
@@ -357,17 +358,22 @@ def http_request_response(  # pylint: disable=,R0913, R0914, R0915
         # Send the request
         res = _request(req, GetRunResponse, PATH_GET_RUN)
         if res is None:
-            return Run(run_id, "", "", {})
+            return Run(run_id, "", "", "", {})
 
         return Run(
             run_id,
             res.run.fab_id,
             res.run.fab_version,
+            res.run.fab_hash,
             user_config_from_proto(res.run.override_config),
         )
 
+    def get_fab(fab_hash: str) -> Fab:
+        # Call FleetAPI
+        raise NotImplementedError
+
     try:
         # Yield methods
-        yield (receive, send, create_node, delete_node, get_run)
+        yield (receive, send, create_node, delete_node, get_run, get_fab)
     except Exception as exc:  # pylint: disable=broad-except
         log(ERROR, exc)

flwr/common/config.py

@@ -74,10 +74,15 @@ def get_project_config(project_dir: Union[str, Path]) -> Dict[str, Any]:
     return config
 
 
-def _fuse_dicts(
+def fuse_dicts(
     main_dict: UserConfig,
     override_dict: UserConfig,
 ) -> UserConfig:
+    """Merge a config with the overrides.
+
+    Remove the nesting by adding the nested keys as prefixes separated by dots, and fuse
+    it with the override dict.
+    """
     fused_dict = main_dict.copy()
 
     for key, value in override_dict.items():
@@ -96,7 +101,7 @@ def get_fused_config_from_dir(
     )
     flat_default_config = flatten_dict(default_config)
 
-    return _fuse_dicts(flat_default_config, override_config)
+    return fuse_dicts(flat_default_config, override_config)
 
 
 def get_fused_config(run: Run, flwr_dir: Optional[Path]) -> UserConfig:

flwr/common/record/recordset.py

@@ -15,8 +15,10 @@
 """RecordSet."""
 
 
+from __future__ import annotations
+
 from dataclasses import dataclass
-from typing import Dict, Optional, cast
+from typing import cast
 
 from .configsrecord import ConfigsRecord
 from .metricsrecord import MetricsRecord
@@ -34,9 +36,9 @@ class RecordSetData:
 
     def __init__(
         self,
-        parameters_records: Optional[Dict[str, ParametersRecord]] = None,
-        metrics_records: Optional[Dict[str, MetricsRecord]] = None,
-        configs_records: Optional[Dict[str, ConfigsRecord]] = None,
+        parameters_records: dict[str, ParametersRecord] | None = None,
+        metrics_records: dict[str, MetricsRecord] | None = None,
+        configs_records: dict[str, ConfigsRecord] | None = None,
     ) -> None:
         self.parameters_records = TypedDict[str, ParametersRecord](
             self._check_fn_str, self._check_fn_params
@@ -88,9 +90,9 @@ class RecordSet:
 
     def __init__(
         self,
-        parameters_records: Optional[Dict[str, ParametersRecord]] = None,
-        metrics_records: Optional[Dict[str, MetricsRecord]] = None,
-        configs_records: Optional[Dict[str, ConfigsRecord]] = None,
+        parameters_records: dict[str, ParametersRecord] | None = None,
+        metrics_records: dict[str, MetricsRecord] | None = None,
+        configs_records: dict[str, ConfigsRecord] | None = None,
     ) -> None:
         data = RecordSetData(
             parameters_records=parameters_records,

flwr/common/record/typeddict.py

@@ -15,99 +15,61 @@
 """Typed dict base class for *Records."""
 
 
-from typing import Any, Callable, Dict, Generic, Iterator, Tuple, TypeVar, cast
+from typing import Callable, Dict, Generic, Iterator, MutableMapping, TypeVar, cast
 
 K = TypeVar("K")  # Key type
 V = TypeVar("V")  # Value type
 
 
-class TypedDict(Generic[K, V]):
+class TypedDict(MutableMapping[K, V], Generic[K, V]):
     """Typed dictionary."""
 
     def __init__(
         self, check_key_fn: Callable[[K], None], check_value_fn: Callable[[V], None]
     ):
-        self._data: Dict[K, V] = {}
-        self._check_key_fn = check_key_fn
-        self._check_value_fn = check_value_fn
+        self.__dict__["_check_key_fn"] = check_key_fn
+        self.__dict__["_check_value_fn"] = check_value_fn
+        self.__dict__["_data"] = {}
 
     def __setitem__(self, key: K, value: V) -> None:
         """Set the given key to the given value after type checking."""
         # Check the types of key and value
-        self._check_key_fn(key)
-        self._check_value_fn(value)
+        cast(Callable[[K], None], self.__dict__["_check_key_fn"])(key)
+        cast(Callable[[V], None], self.__dict__["_check_value_fn"])(value)
+
         # Set key-value pair
-        self._data[key] = value
+        cast(Dict[K, V], self.__dict__["_data"])[key] = value
 
     def __delitem__(self, key: K) -> None:
         """Remove the item with the specified key."""
-        del self._data[key]
+        del cast(Dict[K, V], self.__dict__["_data"])[key]
 
     def __getitem__(self, item: K) -> V:
         """Return the value for the specified key."""
-        return self._data[item]
+        return cast(Dict[K, V], self.__dict__["_data"])[item]
 
     def __iter__(self) -> Iterator[K]:
         """Yield an iterator over the keys of the dictionary."""
-        return iter(self._data)
+        return iter(cast(Dict[K, V], self.__dict__["_data"]))
 
     def __repr__(self) -> str:
         """Return a string representation of the dictionary."""
-        return self._data.__repr__()
+        return cast(Dict[K, V], self.__dict__["_data"]).__repr__()
 
     def __len__(self) -> int:
         """Return the number of items in the dictionary."""
-        return len(self._data)
+        return len(cast(Dict[K, V], self.__dict__["_data"]))
 
-    def __contains__(self, key: K) -> bool:
+    def __contains__(self, key: object) -> bool:
         """Check if the dictionary contains the specified key."""
-        return key in self._data
+        return key in cast(Dict[K, V], self.__dict__["_data"])
 
     def __eq__(self, other: object) -> bool:
         """Compare this instance to another dictionary or TypedDict."""
+        data = cast(Dict[K, V], self.__dict__["_data"])
         if isinstance(other, TypedDict):
-            return self._data == other._data
+            other_data = cast(Dict[K, V], other.__dict__["_data"])
+            return data == other_data
         if isinstance(other, dict):
-            return self._data == other
+            return data == other
         return NotImplemented
-
-    def items(self) -> Iterator[Tuple[K, V]]:
-        """R.items() -> a set-like object providing a view on R's items."""
-        return cast(Iterator[Tuple[K, V]], self._data.items())
-
-    def keys(self) -> Iterator[K]:
-        """R.keys() -> a set-like object providing a view on R's keys."""
-        return cast(Iterator[K], self._data.keys())
-
-    def values(self) -> Iterator[V]:
-        """R.values() -> an object providing a view on R's values."""
-        return cast(Iterator[V], self._data.values())
-
-    def update(self, *args: Any, **kwargs: Any) -> None:
-        """R.update([E, ]**F) -> None.
-
-        Update R from dict/iterable E and F.
-        """
-        for key, value in dict(*args, **kwargs).items():
-            self[key] = value
-
-    def pop(self, key: K) -> V:
-        """R.pop(k[,d]) -> v, remove specified key and return the corresponding value.
-
-        If key is not found, d is returned if given, otherwise KeyError is raised.
-        """
-        return self._data.pop(key)
-
-    def get(self, key: K, default: V) -> V:
-        """R.get(k[,d]) -> R[k] if k in R, else d.
-
-        d defaults to None.
-        """
-        return self._data.get(key, default)
-
-    def clear(self) -> None:
-        """R.clear() -> None.
-
-        Remove all items from R.
-        """
-        self._data.clear()

flwr/common/recordset_compat.py

@@ -145,7 +145,7 @@ def _recordset_to_fit_or_evaluate_ins_components(
     # get config dict
     config_record = recordset.configs_records[f"{ins_str}.config"]
     # pylint: disable-next=protected-access
-    config_dict = _check_mapping_from_recordscalartype_to_scalar(config_record._data)
+    config_dict = _check_mapping_from_recordscalartype_to_scalar(config_record)
 
     return parameters, config_dict
 
@@ -213,7 +213,7 @@ def recordset_to_fitres(recordset: RecordSet, keep_input: bool) -> FitRes:
     )
     configs_record = recordset.configs_records[f"{ins_str}.metrics"]
     # pylint: disable-next=protected-access
-    metrics = _check_mapping_from_recordscalartype_to_scalar(configs_record._data)
+    metrics = _check_mapping_from_recordscalartype_to_scalar(configs_record)
     status = _extract_status_from_recordset(ins_str, recordset)
 
     return FitRes(
@@ -274,7 +274,7 @@ def recordset_to_evaluateres(recordset: RecordSet) -> EvaluateRes:
     configs_record = recordset.configs_records[f"{ins_str}.metrics"]
 
     # pylint: disable-next=protected-access
-    metrics = _check_mapping_from_recordscalartype_to_scalar(configs_record._data)
+    metrics = _check_mapping_from_recordscalartype_to_scalar(configs_record)
     status = _extract_status_from_recordset(ins_str, recordset)
 
     return EvaluateRes(
@@ -314,7 +314,7 @@ def recordset_to_getparametersins(recordset: RecordSet) -> GetParametersIns:
     """Derive GetParametersIns from a RecordSet object."""
     config_record = recordset.configs_records["getparametersins.config"]
     # pylint: disable-next=protected-access
-    config_dict = _check_mapping_from_recordscalartype_to_scalar(config_record._data)
+    config_dict = _check_mapping_from_recordscalartype_to_scalar(config_record)
 
     return GetParametersIns(config=config_dict)
 
@@ -365,7 +365,7 @@ def recordset_to_getpropertiesins(recordset: RecordSet) -> GetPropertiesIns:
     """Derive GetPropertiesIns from a RecordSet object."""
     config_record = recordset.configs_records["getpropertiesins.config"]
     # pylint: disable-next=protected-access
-    config_dict = _check_mapping_from_recordscalartype_to_scalar(config_record._data)
+    config_dict = _check_mapping_from_recordscalartype_to_scalar(config_record)
 
     return GetPropertiesIns(config=config_dict)
 
@@ -384,7 +384,7 @@ def recordset_to_getpropertiesres(recordset: RecordSet) -> GetPropertiesRes:
     res_str = "getpropertiesres"
     config_record = recordset.configs_records[f"{res_str}.properties"]
     # pylint: disable-next=protected-access
-    properties = _check_mapping_from_recordscalartype_to_scalar(config_record._data)
+    properties = _check_mapping_from_recordscalartype_to_scalar(config_record)
 
     status = _extract_status_from_recordset(res_str, recordset=recordset)