flwr-nightly 1.10.0.dev20240624__py3-none-any.whl → 1.10.0.dev20240722__py3-none-any.whl

flwr/client/message_handler/message_handler.py

@@ -14,7 +14,6 @@
 # ==============================================================================
 """Client-side message handler."""
 
-
 from logging import WARN
 from typing import Optional, Tuple, cast
 
@@ -25,7 +24,7 @@ from flwr.client.client import (
     maybe_call_get_properties,
 )
 from flwr.client.numpy_client import NumPyClient
-from flwr.client.typing import ClientFn
+from flwr.client.typing import ClientFnExt
 from flwr.common import ConfigsRecord, Context, Message, Metadata, RecordSet, log
 from flwr.common.constant import MessageType, MessageTypeLegacy
 from flwr.common.recordset_compat import (
@@ -90,10 +89,10 @@ def handle_control_message(message: Message) -> Tuple[Optional[Message], int]:
 
 
 def handle_legacy_message_from_msgtype(
-    client_fn: ClientFn, message: Message, context: Context
+    client_fn: ClientFnExt, message: Message, context: Context
 ) -> Message:
     """Handle legacy message in the inner most mod."""
-    client = client_fn(str(message.metadata.partition_id))
+    client = client_fn(context)
 
     # Check if NumPyClient is returend
     if isinstance(client, NumPyClient):

flwr/client/node_state.py

@@ -15,27 +15,72 @@
 """Node state."""
 
 
-from typing import Any, Dict
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Optional
 
 from flwr.common import Context, RecordSet
+from flwr.common.config import get_fused_config, get_fused_config_from_dir
+from flwr.common.typing import Run, UserConfig
+
+
+@dataclass()
+class RunInfo:
+    """Contains the Context and initial run_config of a Run."""
+
+    context: Context
+    initial_run_config: UserConfig
 
 
 class NodeState:
     """State of a node where client nodes execute runs."""
 
-    def __init__(self) -> None:
-        self._meta: Dict[str, Any] = {}  # holds metadata about the node
-        self.run_contexts: Dict[int, Context] = {}
+    def __init__(
+        self,
+        node_id: int,
+        node_config: UserConfig,
+    ) -> None:
+        self.node_id = node_id
+        self.node_config = node_config
+        self.run_infos: Dict[int, RunInfo] = {}
 
-    def register_context(self, run_id: int) -> None:
+    def register_context(
+        self,
+        run_id: int,
+        run: Optional[Run] = None,
+        flwr_path: Optional[Path] = None,
+        app_dir: Optional[str] = None,
+    ) -> None:
         """Register new run context for this node."""
-        if run_id not in self.run_contexts:
-            self.run_contexts[run_id] = Context(state=RecordSet())
+        if run_id not in self.run_infos:
+            initial_run_config = {}
+            if app_dir:
+                # Load from app directory
+                app_path = Path(app_dir)
+                if app_path.is_dir():
+                    override_config = run.override_config if run else {}
+                    initial_run_config = get_fused_config_from_dir(
+                        app_path, override_config
+                    )
+                else:
+                    raise ValueError("The specified `app_dir` must be a directory.")
+            else:
+                # Load from .fab
+                initial_run_config = get_fused_config(run, flwr_path) if run else {}
+            self.run_infos[run_id] = RunInfo(
+                initial_run_config=initial_run_config,
+                context=Context(
+                    node_id=self.node_id,
+                    node_config=self.node_config,
+                    state=RecordSet(),
+                    run_config=initial_run_config.copy(),
+                ),
+            )
 
     def retrieve_context(self, run_id: int) -> Context:
         """Get run context given a run_id."""
-        if run_id in self.run_contexts:
-            return self.run_contexts[run_id]
+        if run_id in self.run_infos:
+            return self.run_infos[run_id].context
 
         raise RuntimeError(
             f"Context for run_id={run_id} doesn't exist."
@@ -45,4 +90,9 @@ class NodeState:
 
     def update_context(self, run_id: int, context: Context) -> None:
         """Update run context."""
-        self.run_contexts[run_id] = context
+        if context.run_config != self.run_infos[run_id].initial_run_config:
+            raise ValueError(
+                "The `run_config` field of the `Context` object cannot be "
+                f"modified (run_id: {run_id})."
+            )
+        self.run_infos[run_id].context = context

flwr/client/node_state_tests.py

@@ -41,7 +41,7 @@ def test_multirun_in_node_state() -> None:
     expected_values = {0: "1", 1: "1" * 3, 2: "1" * 2, 3: "1", 5: "1"}
 
     # NodeState
-    node_state = NodeState()
+    node_state = NodeState(node_id=0, node_config={})
 
     for task in tasks:
         run_id = task.run_id
@@ -59,7 +59,8 @@ def test_multirun_in_node_state() -> None:
         node_state.update_context(run_id=run_id, context=updated_state)
 
     # Verify values
-    for run_id, context in node_state.run_contexts.items():
+    for run_id, run_info in node_state.run_infos.items():
         assert (
-            context.state.configs_records["counter"]["count"] == expected_values[run_id]
+            run_info.context.state.configs_records["counter"]["count"]
+            == expected_values[run_id]
         )

flwr/client/rest_client/connection.py

@@ -40,7 +40,12 @@ from flwr.common.constant import (
 from flwr.common.logger import log
 from flwr.common.message import Message, Metadata
 from flwr.common.retry_invoker import RetryInvoker
-from flwr.common.serde import message_from_taskins, message_to_taskres
+from flwr.common.serde import (
+    message_from_taskins,
+    message_to_taskres,
+    user_config_from_proto,
+)
+from flwr.common.typing import Run
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     CreateNodeResponse,
@@ -89,9 +94,9 @@ def http_request_response(  # pylint: disable=,R0913, R0914, R0915
     Tuple[
         Callable[[], Optional[Message]],
         Callable[[Message], None],
+        Optional[Callable[[], Optional[int]]],
         Optional[Callable[[], None]],
-        Optional[Callable[[], None]],
-        Optional[Callable[[int], Tuple[str, str]]],
+        Optional[Callable[[int], Run]],
     ]
 ]:
     """Primitives for request/response-based interaction with a server.
@@ -236,19 +241,20 @@ def http_request_response(  # pylint: disable=,R0913, R0914, R0915
         if not ping_stop_event.is_set():
             ping_stop_event.wait(next_interval)
 
-    def create_node() -> None:
+    def create_node() -> Optional[int]:
         """Set create_node."""
         req = CreateNodeRequest(ping_interval=PING_DEFAULT_INTERVAL)
 
         # Send the request
         res = _request(req, CreateNodeResponse, PATH_CREATE_NODE)
         if res is None:
-            return
+            return None
 
         # Remember the node and the ping-loop thread
         nonlocal node, ping_thread
         node = res.node
         ping_thread = start_ping_loop(ping, ping_stop_event)
+        return node.node_id
 
     def delete_node() -> None:
         """Set delete_node."""
@@ -344,16 +350,21 @@ def http_request_response(  # pylint: disable=,R0913, R0914, R0915
             res.results,  # pylint: disable=no-member
         )
 
-    def get_run(run_id: int) -> Tuple[str, str]:
+    def get_run(run_id: int) -> Run:
         # Construct the request
         req = GetRunRequest(run_id=run_id)
 
         # Send the request
         res = _request(req, GetRunResponse, PATH_GET_RUN)
         if res is None:
-            return "", ""
+            return Run(run_id, "", "", {})
 
-        return res.run.fab_id, res.run.fab_version
+        return Run(
+            run_id,
+            res.run.fab_id,
+            res.run.fab_version,
+            user_config_from_proto(res.run.override_config),
+        )
 
     try:
         # Yield methods

flwr/client/supernode/app.py

@@ -29,7 +29,12 @@ from cryptography.hazmat.primitives.serialization import (
 
 from flwr.client.client_app import ClientApp, LoadClientAppError
 from flwr.common import EventType, event
-from flwr.common.config import get_flwr_dir, get_project_config, get_project_dir
+from flwr.common.config import (
+    get_flwr_dir,
+    get_project_config,
+    get_project_dir,
+    parse_config_args,
+)
 from flwr.common.constant import (
     TRANSPORT_TYPE_GRPC_ADAPTER,
     TRANSPORT_TYPE_GRPC_RERE,
@@ -55,7 +60,12 @@ def run_supernode() -> None:
     _warn_deprecated_server_arg(args)
 
     root_certificates = _get_certificates(args)
-    load_fn = _get_load_client_app_fn(args, multi_app=True)
+    load_fn = _get_load_client_app_fn(
+        default_app_ref=getattr(args, "client-app"),
+        dir_arg=args.dir,
+        flwr_dir_arg=args.flwr_dir,
+        multi_app=True,
+    )
     authentication_keys = _try_setup_client_authentication(args)
 
     _start_client_internal(
@@ -67,6 +77,8 @@ def run_supernode() -> None:
         authentication_keys=authentication_keys,
         max_retries=args.max_retries,
         max_wait_time=args.max_wait_time,
+        node_config=parse_config_args([args.node_config]),
+        flwr_path=get_flwr_dir(args.flwr_dir),
     )
 
     # Graceful shutdown
@@ -86,11 +98,16 @@ def run_client_app() -> None:
     _warn_deprecated_server_arg(args)
 
     root_certificates = _get_certificates(args)
-    load_fn = _get_load_client_app_fn(args, multi_app=False)
+    load_fn = _get_load_client_app_fn(
+        default_app_ref=getattr(args, "client-app"),
+        dir_arg=args.dir,
+        multi_app=False,
+    )
     authentication_keys = _try_setup_client_authentication(args)
 
     _start_client_internal(
         server_address=args.superlink,
+        node_config=parse_config_args([args.node_config]),
         load_client_app_fn=load_fn,
         transport=args.transport,
         root_certificates=root_certificates,
@@ -158,7 +175,10 @@ def _get_certificates(args: argparse.Namespace) -> Optional[bytes]:
 
 
 def _get_load_client_app_fn(
-    args: argparse.Namespace, multi_app: bool
+    default_app_ref: str,
+    dir_arg: str,
+    multi_app: bool,
+    flwr_dir_arg: Optional[str] = None,
 ) -> Callable[[str, str], ClientApp]:
     """Get the load_client_app_fn function.
 
@@ -170,23 +190,27 @@ def _get_load_client_app_fn(
     loads a default ClientApp.
     """
     # Find the Flower directory containing Flower Apps (only for multi-app)
-    flwr_dir = Path("")
-    if "flwr_dir" in args:
-        if args.flwr_dir is None:
+    if not multi_app:
+        flwr_dir = Path("")
+    else:
+        if flwr_dir_arg is None:
             flwr_dir = get_flwr_dir()
         else:
-            flwr_dir = Path(args.flwr_dir).absolute()
-
-    sys.path.insert(0, str(flwr_dir.absolute()))
+            flwr_dir = Path(flwr_dir_arg).absolute()
 
-    default_app_ref: str = getattr(args, "client-app")
+    inserted_path = None
 
     if not multi_app:
         log(
             DEBUG,
             "Flower SuperNode will load and validate ClientApp `%s`",
-            getattr(args, "client-app"),
+            default_app_ref,
         )
+        # Insert sys.path
+        dir_path = Path(dir_arg).absolute()
+        sys.path.insert(0, str(dir_path))
+        inserted_path = str(dir_path)
+
         valid, error_msg = validate(default_app_ref)
         if not valid and error_msg:
             raise LoadClientAppError(error_msg) from None
@@ -195,7 +219,7 @@ def _get_load_client_app_fn(
         # If multi-app feature is disabled
         if not multi_app:
             # Get sys path to be inserted
-            sys_path = Path(args.dir).absolute()
+            dir_path = Path(dir_arg).absolute()
 
             # Set app reference
             client_app_ref = default_app_ref
@@ -208,7 +232,7 @@ def _get_load_client_app_fn(
 
             log(WARN, "FAB ID is not provided; the default ClientApp will be loaded.")
             # Get sys path to be inserted
-            sys_path = Path(args.dir).absolute()
+            dir_path = Path(dir_arg).absolute()
 
             # Set app reference
             client_app_ref = default_app_ref
@@ -221,13 +245,21 @@ def _get_load_client_app_fn(
                 raise LoadClientAppError("Failed to load ClientApp") from e
 
             # Get sys path to be inserted
-            sys_path = Path(project_dir).absolute()
+            dir_path = Path(project_dir).absolute()
 
             # Set app reference
-            client_app_ref = config["flower"]["components"]["clientapp"]
+            client_app_ref = config["tool"]["flwr"]["app"]["components"]["clientapp"]
 
         # Set sys.path
-        sys.path.insert(0, str(sys_path))
+        nonlocal inserted_path
+        if inserted_path != str(dir_path):
+            # Remove the previously inserted path
+            if inserted_path is not None:
+                sys.path.remove(inserted_path)
+            # Insert the new path
+            sys.path.insert(0, str(dir_path))
+
+        inserted_path = str(dir_path)
 
         # Load ClientApp
         log(
@@ -235,7 +267,7 @@ def _get_load_client_app_fn(
             "Loading ClientApp `%s`",
             client_app_ref,
         )
-        client_app = load_app(client_app_ref, LoadClientAppError, sys_path)
+        client_app = load_app(client_app_ref, LoadClientAppError, dir_path)
 
         if not isinstance(client_app, ClientApp):
             raise LoadClientAppError(
@@ -344,8 +376,8 @@ def _parse_args_common(parser: argparse.ArgumentParser) -> None:
         "--max-retries",
         type=int,
         default=None,
-        help="The maximum number of times the client will try to connect to the"
-        "server before giving up in case of a connection error. By default,"
+        help="The maximum number of times the client will try to reconnect to the"
+        "SuperLink before giving up in case of a connection error. By default,"
         "it is set to None, meaning there is no limit to the number of tries.",
     )
     parser.add_argument(
@@ -353,7 +385,7 @@ def _parse_args_common(parser: argparse.ArgumentParser) -> None:
         type=float,
         default=None,
         help="The maximum duration before the client stops trying to"
-        "connect to the server in case of connection error. By default, it"
+        "connect to the SuperLink in case of connection error. By default, it"
         "is set to None, meaning there is no limit to the total time.",
     )
     parser.add_argument(
@@ -373,6 +405,13 @@ def _parse_args_common(parser: argparse.ArgumentParser) -> None:
         type=str,
         help="The SuperNode's public key (as a path str) to enable authentication.",
     )
+    parser.add_argument(
+        "--node-config",
+        type=str,
+        help="A comma separated list of key/value pairs (separated by `=`) to "
+        "configure the SuperNode. "
+        "E.g. --node-config 'key1=\"value1\",partition-id=0,num-partitions=100'",
+    )
 
 
 def _try_setup_client_authentication(

flwr/client/typing.py

@@ -23,6 +23,7 @@ from .client import Client as Client
 
 # Compatibility
 ClientFn = Callable[[str], Client]
+ClientFnExt = Callable[[Context], Client]
 
 ClientAppCallable = Callable[[Message, Context], Message]
 Mod = Callable[[Message, Context, ClientAppCallable], Message]

flwr/common/config.py

@@ -16,12 +16,13 @@
 
 import os
 from pathlib import Path
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, List, Optional, Tuple, Union, cast, get_args
 
 import tomli
 
 from flwr.cli.config_utils import validate_fields
 from flwr.common.constant import APP_DIR, FAB_CONFIG_FILE, FLWR_HOME
+from flwr.common.typing import Run, UserConfig, UserConfigValue
 
 
 def get_flwr_dir(provided_path: Optional[str] = None) -> Path:
@@ -30,7 +31,7 @@ def get_flwr_dir(provided_path: Optional[str] = None) -> Path:
         return Path(
             os.getenv(
                 FLWR_HOME,
-                f"{os.getenv('XDG_DATA_HOME', os.getenv('HOME'))}/.flwr",
+                Path(f"{os.getenv('XDG_DATA_HOME', os.getenv('HOME'))}") / ".flwr",
             )
         )
     return Path(provided_path).absolute()
@@ -71,3 +72,87 @@ def get_project_config(project_dir: Union[str, Path]) -> Dict[str, Any]:
         )
 
     return config
+
+
+def _fuse_dicts(
+    main_dict: UserConfig,
+    override_dict: UserConfig,
+) -> UserConfig:
+    fused_dict = main_dict.copy()
+
+    for key, value in override_dict.items():
+        if key in main_dict:
+            fused_dict[key] = value
+
+    return fused_dict
+
+
+def get_fused_config_from_dir(
+    project_dir: Path, override_config: UserConfig
+) -> UserConfig:
+    """Merge the overrides from a given dict with the config from a Flower App."""
+    default_config = get_project_config(project_dir)["tool"]["flwr"]["app"].get(
+        "config", {}
+    )
+    flat_default_config = flatten_dict(default_config)
+
+    return _fuse_dicts(flat_default_config, override_config)
+
+
+def get_fused_config(run: Run, flwr_dir: Optional[Path]) -> UserConfig:
+    """Merge the overrides from a `Run` with the config from a FAB.
+
+    Get the config using the fab_id and the fab_version, remove the nesting by adding
+    the nested keys as prefixes separated by dots, and fuse it with the override dict.
+    """
+    if not run.fab_id or not run.fab_version:
+        return {}
+
+    project_dir = get_project_dir(run.fab_id, run.fab_version, flwr_dir)
+
+    return get_fused_config_from_dir(project_dir, run.override_config)
+
+
+def flatten_dict(raw_dict: Dict[str, Any], parent_key: str = "") -> UserConfig:
+    """Flatten dict by joining nested keys with a given separator."""
+    items: List[Tuple[str, UserConfigValue]] = []
+    separator: str = "."
+    for k, v in raw_dict.items():
+        new_key = f"{parent_key}{separator}{k}" if parent_key else k
+        if isinstance(v, dict):
+            items.extend(flatten_dict(v, parent_key=new_key).items())
+        elif isinstance(v, get_args(UserConfigValue)):
+            items.append((new_key, cast(UserConfigValue, v)))
+        else:
+            raise ValueError(
+                f"The value for key {k} needs to be of type `int`, `float`, "
+                "`bool, `str`, or  a `dict` of those.",
+            )
+    return dict(items)
+
+
+def parse_config_args(
+    config: Optional[List[str]],
+    separator: str = ",",
+) -> UserConfig:
+    """Parse separator separated list of key-value pairs separated by '='."""
+    overrides: UserConfig = {}
+
+    if config is None:
+        return overrides
+
+    for config_line in config:
+        if config_line:
+            overrides_list = config_line.split(separator)
+            if (
+                len(overrides_list) == 1
+                and "=" not in overrides_list
+                and overrides_list[0].endswith(".toml")
+            ):
+                with Path(overrides_list[0]).open("rb") as config_file:
+                    overrides = flatten_dict(tomli.load(config_file))
+            else:
+                toml_str = "\n".join(overrides_list)
+                overrides.update(tomli.loads(toml_str))
+
+    return overrides

flwr/common/constant.py

@@ -46,6 +46,9 @@ PING_BASE_MULTIPLIER = 0.8
 PING_RANDOM_RANGE = (-0.1, 0.1)
 PING_MAX_INTERVAL = 1e300
 
+# IDs
+RUN_ID_NUM_BYTES = 8
+NODE_ID_NUM_BYTES = 8
 GRPC_ADAPTER_METADATA_FLOWER_VERSION_KEY = "flower-version"
 GRPC_ADAPTER_METADATA_SHOULD_EXIT_KEY = "should-exit"
 
@@ -54,6 +57,9 @@ APP_DIR = "apps"
 FAB_CONFIG_FILE = "pyproject.toml"
 FLWR_HOME = "FLWR_HOME"
 
+# Constants entries in Node config for Simulation
+PARTITION_ID_KEY = "partition-id"
+NUM_PARTITIONS_KEY = "num-partitions"
 
 GRPC_ADAPTER_METADATA_FLOWER_VERSION_KEY = "flower-version"
 GRPC_ADAPTER_METADATA_SHOULD_EXIT_KEY = "should-exit"

flwr/common/context.py

@@ -18,14 +18,20 @@
 from dataclasses import dataclass
 
 from .record import RecordSet
+from .typing import UserConfig
 
 
 @dataclass
 class Context:
-    """State of your run.
+    """Context of your run.
 
     Parameters
     ----------
+    node_id : int
+        The ID that identifies the node.
+    node_config : UserConfig
+        A config (key/value mapping) unique to the node and independent of the
+        `run_config`. This config persists across all runs this node participates in.
     state : RecordSet
         Holds records added by the entity in a given run and that will stay local.
         This means that the data it holds will never leave the system it's running from.
@@ -33,6 +39,25 @@ class Context:
         executing mods. It can also be used as a memory to access
         at different points during the lifecycle of this entity (e.g. across
         multiple rounds)
+    run_config : UserConfig
+        A config (key/value mapping) held by the entity in a given run and that will
+        stay local. It can be used at any point during the lifecycle of this entity
+        (e.g. across multiple rounds)
     """
 
+    node_id: int
+    node_config: UserConfig
     state: RecordSet
+    run_config: UserConfig
+
+    def __init__(  # pylint: disable=too-many-arguments
+        self,
+        node_id: int,
+        node_config: UserConfig,
+        state: RecordSet,
+        run_config: UserConfig,
+    ) -> None:
+        self.node_id = node_id
+        self.node_config = node_config
+        self.state = state
+        self.run_config = run_config

flwr/common/logger.py

@@ -197,6 +197,44 @@ def warn_deprecated_feature(name: str) -> None:
     )
 
 
+def warn_deprecated_feature_with_example(
+    deprecation_message: str, example_message: str, code_example: str
+) -> None:
+    """Warn if a feature is deprecated and show code example."""
+    log(
+        WARN,
+        """DEPRECATED FEATURE: %s
+
+            Check the following `FEATURE UPDATE` warning message for the preferred
+            new mechanism to use this feature in Flower.
+        """,
+        deprecation_message,
+    )
+    log(
+        WARN,
+        """FEATURE UPDATE: %s
+        ------------------------------------------------------------
+        %s
+        ------------------------------------------------------------
+        """,
+        example_message,
+        code_example,
+    )
+
+
+def warn_unsupported_feature(name: str) -> None:
+    """Warn the user when they use an unsupported feature."""
+    log(
+        WARN,
+        """UNSUPPORTED FEATURE: %s
+
+            This is an unsupported feature. It will be removed
+            entirely in future versions of Flower.
+        """,
+        name,
+    )
+
+
 def set_logger_propagation(
     child_logger: logging.Logger, value: bool = True
 ) -> logging.Logger:

flwr/common/message.py

@@ -48,10 +48,6 @@ class Metadata:  # pylint: disable=too-many-instance-attributes
     message_type : str
         A string that encodes the action to be executed on
         the receiving end.
-    partition_id : Optional[int]
-        An identifier that can be used when loading a particular
-        data partition for a ClientApp. Making use of this identifier
-        is more relevant when conducting simulations.
     """
 
     def __init__(  # pylint: disable=too-many-arguments
@@ -64,7 +60,6 @@ class Metadata:  # pylint: disable=too-many-instance-attributes
         group_id: str,
         ttl: float,
         message_type: str,
-        partition_id: int | None = None,
     ) -> None:
         var_dict = {
             "_run_id": run_id,
@@ -75,7 +70,6 @@ class Metadata:  # pylint: disable=too-many-instance-attributes
             "_group_id": group_id,
             "_ttl": ttl,
             "_message_type": message_type,
-            "_partition_id": partition_id,
         }
         self.__dict__.update(var_dict)
 
@@ -149,16 +143,6 @@ class Metadata:  # pylint: disable=too-many-instance-attributes
         """Set message_type."""
         self.__dict__["_message_type"] = value
 
-    @property
-    def partition_id(self) -> int | None:
-        """An identifier telling which data partition a ClientApp should use."""
-        return cast(int, self.__dict__["_partition_id"])
-
-    @partition_id.setter
-    def partition_id(self, value: int) -> None:
-        """Set partition_id."""
-        self.__dict__["_partition_id"] = value
-
     def __repr__(self) -> str:
         """Return a string representation of this instance."""
         view = ", ".join([f"{k.lstrip('_')}={v!r}" for k, v in self.__dict__.items()])
@@ -398,5 +382,4 @@ def _create_reply_metadata(msg: Message, ttl: float) -> Metadata:
         group_id=msg.metadata.group_id,
         ttl=ttl,
         message_type=msg.metadata.message_type,
-        partition_id=msg.metadata.partition_id,
     )