flwr-nightly 1.10.0.dev20240707__py3-none-any.whl → 1.10.0.dev20240722__py3-none-any.whl

flwr/client/node_state.py

@@ -15,30 +15,72 @@
 """Node state."""
 
 
-from typing import Any, Dict, Optional
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Optional
 
 from flwr.common import Context, RecordSet
+from flwr.common.config import get_fused_config, get_fused_config_from_dir
+from flwr.common.typing import Run, UserConfig
+
+
+@dataclass()
+class RunInfo:
+    """Contains the Context and initial run_config of a Run."""
+
+    context: Context
+    initial_run_config: UserConfig
 
 
 class NodeState:
     """State of a node where client nodes execute runs."""
 
-    def __init__(self, partition_id: Optional[int]) -> None:
-        self._meta: Dict[str, Any] = {}  # holds metadata about the node
-        self.run_contexts: Dict[int, Context] = {}
-        self._partition_id = partition_id
+    def __init__(
+        self,
+        node_id: int,
+        node_config: UserConfig,
+    ) -> None:
+        self.node_id = node_id
+        self.node_config = node_config
+        self.run_infos: Dict[int, RunInfo] = {}
 
-    def register_context(self, run_id: int) -> None:
+    def register_context(
+        self,
+        run_id: int,
+        run: Optional[Run] = None,
+        flwr_path: Optional[Path] = None,
+        app_dir: Optional[str] = None,
+    ) -> None:
         """Register new run context for this node."""
-        if run_id not in self.run_contexts:
-            self.run_contexts[run_id] = Context(
-                state=RecordSet(), partition_id=self._partition_id
+        if run_id not in self.run_infos:
+            initial_run_config = {}
+            if app_dir:
+                # Load from app directory
+                app_path = Path(app_dir)
+                if app_path.is_dir():
+                    override_config = run.override_config if run else {}
+                    initial_run_config = get_fused_config_from_dir(
+                        app_path, override_config
+                    )
+                else:
+                    raise ValueError("The specified `app_dir` must be a directory.")
+            else:
+                # Load from .fab
+                initial_run_config = get_fused_config(run, flwr_path) if run else {}
+            self.run_infos[run_id] = RunInfo(
+                initial_run_config=initial_run_config,
+                context=Context(
+                    node_id=self.node_id,
+                    node_config=self.node_config,
+                    state=RecordSet(),
+                    run_config=initial_run_config.copy(),
+                ),
             )
 
     def retrieve_context(self, run_id: int) -> Context:
         """Get run context given a run_id."""
-        if run_id in self.run_contexts:
-            return self.run_contexts[run_id]
+        if run_id in self.run_infos:
+            return self.run_infos[run_id].context
 
         raise RuntimeError(
             f"Context for run_id={run_id} doesn't exist."
@@ -48,4 +90,9 @@ class NodeState:
 
     def update_context(self, run_id: int, context: Context) -> None:
         """Update run context."""
-        self.run_contexts[run_id] = context
+        if context.run_config != self.run_infos[run_id].initial_run_config:
+            raise ValueError(
+                "The `run_config` field of the `Context` object cannot be "
+                f"modified (run_id: {run_id})."
+            )
+        self.run_infos[run_id].context = context

flwr/client/node_state_tests.py

@@ -41,7 +41,7 @@ def test_multirun_in_node_state() -> None:
     expected_values = {0: "1", 1: "1" * 3, 2: "1" * 2, 3: "1", 5: "1"}
 
     # NodeState
-    node_state = NodeState(partition_id=None)
+    node_state = NodeState(node_id=0, node_config={})
 
     for task in tasks:
         run_id = task.run_id
@@ -59,7 +59,8 @@ def test_multirun_in_node_state() -> None:
         node_state.update_context(run_id=run_id, context=updated_state)
 
     # Verify values
-    for run_id, context in node_state.run_contexts.items():
+    for run_id, run_info in node_state.run_infos.items():
         assert (
-            context.state.configs_records["counter"]["count"] == expected_values[run_id]
+            run_info.context.state.configs_records["counter"]["count"]
+            == expected_values[run_id]
         )

flwr/client/rest_client/connection.py

@@ -40,7 +40,12 @@ from flwr.common.constant import (
 from flwr.common.logger import log
 from flwr.common.message import Message, Metadata
 from flwr.common.retry_invoker import RetryInvoker
-from flwr.common.serde import message_from_taskins, message_to_taskres
+from flwr.common.serde import (
+    message_from_taskins,
+    message_to_taskres,
+    user_config_from_proto,
+)
+from flwr.common.typing import Run
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     CreateNodeResponse,
@@ -89,9 +94,9 @@ def http_request_response(  # pylint: disable=,R0913, R0914, R0915
     Tuple[
         Callable[[], Optional[Message]],
         Callable[[Message], None],
+        Optional[Callable[[], Optional[int]]],
         Optional[Callable[[], None]],
-        Optional[Callable[[], None]],
-        Optional[Callable[[int], Tuple[str, str]]],
+        Optional[Callable[[int], Run]],
     ]
 ]:
     """Primitives for request/response-based interaction with a server.
@@ -236,19 +241,20 @@ def http_request_response(  # pylint: disable=,R0913, R0914, R0915
         if not ping_stop_event.is_set():
             ping_stop_event.wait(next_interval)
 
-    def create_node() -> None:
+    def create_node() -> Optional[int]:
         """Set create_node."""
         req = CreateNodeRequest(ping_interval=PING_DEFAULT_INTERVAL)
 
         # Send the request
         res = _request(req, CreateNodeResponse, PATH_CREATE_NODE)
         if res is None:
-            return
+            return None
 
         # Remember the node and the ping-loop thread
         nonlocal node, ping_thread
         node = res.node
         ping_thread = start_ping_loop(ping, ping_stop_event)
+        return node.node_id
 
     def delete_node() -> None:
         """Set delete_node."""
@@ -344,16 +350,21 @@ def http_request_response(  # pylint: disable=,R0913, R0914, R0915
             res.results,  # pylint: disable=no-member
         )
 
-    def get_run(run_id: int) -> Tuple[str, str]:
+    def get_run(run_id: int) -> Run:
         # Construct the request
         req = GetRunRequest(run_id=run_id)
 
         # Send the request
         res = _request(req, GetRunResponse, PATH_GET_RUN)
         if res is None:
-            return "", ""
+            return Run(run_id, "", "", {})
 
-        return res.run.fab_id, res.run.fab_version
+        return Run(
+            run_id,
+            res.run.fab_id,
+            res.run.fab_version,
+            user_config_from_proto(res.run.override_config),
+        )
 
     try:
         # Yield methods

flwr/client/supernode/app.py

@@ -29,7 +29,12 @@ from cryptography.hazmat.primitives.serialization import (
 
 from flwr.client.client_app import ClientApp, LoadClientAppError
 from flwr.common import EventType, event
-from flwr.common.config import get_flwr_dir, get_project_config, get_project_dir
+from flwr.common.config import (
+    get_flwr_dir,
+    get_project_config,
+    get_project_dir,
+    parse_config_args,
+)
 from flwr.common.constant import (
     TRANSPORT_TYPE_GRPC_ADAPTER,
     TRANSPORT_TYPE_GRPC_RERE,
@@ -55,7 +60,12 @@ def run_supernode() -> None:
     _warn_deprecated_server_arg(args)
 
     root_certificates = _get_certificates(args)
-    load_fn = _get_load_client_app_fn(args, multi_app=True)
+    load_fn = _get_load_client_app_fn(
+        default_app_ref=getattr(args, "client-app"),
+        dir_arg=args.dir,
+        flwr_dir_arg=args.flwr_dir,
+        multi_app=True,
+    )
     authentication_keys = _try_setup_client_authentication(args)
 
     _start_client_internal(
@@ -67,7 +77,8 @@ def run_supernode() -> None:
         authentication_keys=authentication_keys,
         max_retries=args.max_retries,
         max_wait_time=args.max_wait_time,
-        partition_id=args.partition_id,
+        node_config=parse_config_args([args.node_config]),
+        flwr_path=get_flwr_dir(args.flwr_dir),
     )
 
     # Graceful shutdown
@@ -87,11 +98,16 @@ def run_client_app() -> None:
     _warn_deprecated_server_arg(args)
 
     root_certificates = _get_certificates(args)
-    load_fn = _get_load_client_app_fn(args, multi_app=False)
+    load_fn = _get_load_client_app_fn(
+        default_app_ref=getattr(args, "client-app"),
+        dir_arg=args.dir,
+        multi_app=False,
+    )
     authentication_keys = _try_setup_client_authentication(args)
 
     _start_client_internal(
         server_address=args.superlink,
+        node_config=parse_config_args([args.node_config]),
         load_client_app_fn=load_fn,
         transport=args.transport,
         root_certificates=root_certificates,
@@ -159,7 +175,10 @@ def _get_certificates(args: argparse.Namespace) -> Optional[bytes]:
 
 
 def _get_load_client_app_fn(
-    args: argparse.Namespace, multi_app: bool
+    default_app_ref: str,
+    dir_arg: str,
+    multi_app: bool,
+    flwr_dir_arg: Optional[str] = None,
 ) -> Callable[[str, str], ClientApp]:
     """Get the load_client_app_fn function.
 
@@ -171,23 +190,27 @@ def _get_load_client_app_fn(
     loads a default ClientApp.
     """
     # Find the Flower directory containing Flower Apps (only for multi-app)
-    flwr_dir = Path("")
-    if "flwr_dir" in args:
-        if args.flwr_dir is None:
+    if not multi_app:
+        flwr_dir = Path("")
+    else:
+        if flwr_dir_arg is None:
             flwr_dir = get_flwr_dir()
         else:
-            flwr_dir = Path(args.flwr_dir).absolute()
+            flwr_dir = Path(flwr_dir_arg).absolute()
 
-    sys.path.insert(0, str(flwr_dir.absolute()))
-
-    default_app_ref: str = getattr(args, "client-app")
+    inserted_path = None
 
     if not multi_app:
         log(
             DEBUG,
             "Flower SuperNode will load and validate ClientApp `%s`",
-            getattr(args, "client-app"),
+            default_app_ref,
         )
+        # Insert sys.path
+        dir_path = Path(dir_arg).absolute()
+        sys.path.insert(0, str(dir_path))
+        inserted_path = str(dir_path)
+
         valid, error_msg = validate(default_app_ref)
         if not valid and error_msg:
             raise LoadClientAppError(error_msg) from None
@@ -196,7 +219,7 @@ def _get_load_client_app_fn(
         # If multi-app feature is disabled
         if not multi_app:
             # Get sys path to be inserted
-            sys_path = Path(args.dir).absolute()
+            dir_path = Path(dir_arg).absolute()
 
             # Set app reference
             client_app_ref = default_app_ref
@@ -209,7 +232,7 @@ def _get_load_client_app_fn(
 
             log(WARN, "FAB ID is not provided; the default ClientApp will be loaded.")
             # Get sys path to be inserted
-            sys_path = Path(args.dir).absolute()
+            dir_path = Path(dir_arg).absolute()
 
             # Set app reference
             client_app_ref = default_app_ref
@@ -222,13 +245,21 @@ def _get_load_client_app_fn(
                 raise LoadClientAppError("Failed to load ClientApp") from e
 
             # Get sys path to be inserted
-            sys_path = Path(project_dir).absolute()
+            dir_path = Path(project_dir).absolute()
 
             # Set app reference
-            client_app_ref = config["flower"]["components"]["clientapp"]
+            client_app_ref = config["tool"]["flwr"]["app"]["components"]["clientapp"]
 
         # Set sys.path
-        sys.path.insert(0, str(sys_path))
+        nonlocal inserted_path
+        if inserted_path != str(dir_path):
+            # Remove the previously inserted path
+            if inserted_path is not None:
+                sys.path.remove(inserted_path)
+            # Insert the new path
+            sys.path.insert(0, str(dir_path))
+
+        inserted_path = str(dir_path)
 
         # Load ClientApp
         log(
@@ -236,7 +267,7 @@ def _get_load_client_app_fn(
             "Loading ClientApp `%s`",
             client_app_ref,
         )
-        client_app = load_app(client_app_ref, LoadClientAppError, sys_path)
+        client_app = load_app(client_app_ref, LoadClientAppError, dir_path)
 
         if not isinstance(client_app, ClientApp):
             raise LoadClientAppError(
@@ -375,11 +406,11 @@ def _parse_args_common(parser: argparse.ArgumentParser) -> None:
         help="The SuperNode's public key (as a path str) to enable authentication.",
     )
     parser.add_argument(
-        "--partition-id",
-        type=int,
-        help="The data partition index associated with this SuperNode. Better suited "
-        "for prototyping purposes where a SuperNode might only load a fraction of an "
-        "artificially partitioned dataset (e.g. using `flwr-datasets`)",
+        "--node-config",
+        type=str,
+        help="A comma separated list of key/value pairs (separated by `=`) to "
+        "configure the SuperNode. "
+        "E.g. --node-config 'key1=\"value1\",partition-id=0,num-partitions=100'",
     )
 
 

flwr/client/typing.py

@@ -15,7 +15,7 @@
 """Custom types for Flower clients."""
 
 
-from typing import Callable, Optional
+from typing import Callable
 
 from flwr.common import Context, Message
 
@@ -23,7 +23,7 @@ from .client import Client as Client
 
 # Compatibility
 ClientFn = Callable[[str], Client]
-ClientFnExt = Callable[[int, Optional[int]], Client]
+ClientFnExt = Callable[[Context], Client]
 
 ClientAppCallable = Callable[[Message, Context], Message]
 Mod = Callable[[Message, Context, ClientAppCallable], Message]

flwr/common/config.py

@@ -16,12 +16,13 @@
 
 import os
 from pathlib import Path
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, List, Optional, Tuple, Union, cast, get_args
 
 import tomli
 
 from flwr.cli.config_utils import validate_fields
 from flwr.common.constant import APP_DIR, FAB_CONFIG_FILE, FLWR_HOME
+from flwr.common.typing import Run, UserConfig, UserConfigValue
 
 
 def get_flwr_dir(provided_path: Optional[str] = None) -> Path:
@@ -30,7 +31,7 @@ def get_flwr_dir(provided_path: Optional[str] = None) -> Path:
         return Path(
             os.getenv(
                 FLWR_HOME,
-                f"{os.getenv('XDG_DATA_HOME', os.getenv('HOME'))}/.flwr",
+                Path(f"{os.getenv('XDG_DATA_HOME', os.getenv('HOME'))}") / ".flwr",
             )
         )
     return Path(provided_path).absolute()
@@ -71,3 +72,87 @@ def get_project_config(project_dir: Union[str, Path]) -> Dict[str, Any]:
         )
 
     return config
+
+
+def _fuse_dicts(
+    main_dict: UserConfig,
+    override_dict: UserConfig,
+) -> UserConfig:
+    fused_dict = main_dict.copy()
+
+    for key, value in override_dict.items():
+        if key in main_dict:
+            fused_dict[key] = value
+
+    return fused_dict
+
+
+def get_fused_config_from_dir(
+    project_dir: Path, override_config: UserConfig
+) -> UserConfig:
+    """Merge the overrides from a given dict with the config from a Flower App."""
+    default_config = get_project_config(project_dir)["tool"]["flwr"]["app"].get(
+        "config", {}
+    )
+    flat_default_config = flatten_dict(default_config)
+
+    return _fuse_dicts(flat_default_config, override_config)
+
+
+def get_fused_config(run: Run, flwr_dir: Optional[Path]) -> UserConfig:
+    """Merge the overrides from a `Run` with the config from a FAB.
+
+    Get the config using the fab_id and the fab_version, remove the nesting by adding
+    the nested keys as prefixes separated by dots, and fuse it with the override dict.
+    """
+    if not run.fab_id or not run.fab_version:
+        return {}
+
+    project_dir = get_project_dir(run.fab_id, run.fab_version, flwr_dir)
+
+    return get_fused_config_from_dir(project_dir, run.override_config)
+
+
+def flatten_dict(raw_dict: Dict[str, Any], parent_key: str = "") -> UserConfig:
+    """Flatten dict by joining nested keys with a given separator."""
+    items: List[Tuple[str, UserConfigValue]] = []
+    separator: str = "."
+    for k, v in raw_dict.items():
+        new_key = f"{parent_key}{separator}{k}" if parent_key else k
+        if isinstance(v, dict):
+            items.extend(flatten_dict(v, parent_key=new_key).items())
+        elif isinstance(v, get_args(UserConfigValue)):
+            items.append((new_key, cast(UserConfigValue, v)))
+        else:
+            raise ValueError(
+                f"The value for key {k} needs to be of type `int`, `float`, "
+                "`bool, `str`, or  a `dict` of those.",
+            )
+    return dict(items)
+
+
+def parse_config_args(
+    config: Optional[List[str]],
+    separator: str = ",",
+) -> UserConfig:
+    """Parse separator separated list of key-value pairs separated by '='."""
+    overrides: UserConfig = {}
+
+    if config is None:
+        return overrides
+
+    for config_line in config:
+        if config_line:
+            overrides_list = config_line.split(separator)
+            if (
+                len(overrides_list) == 1
+                and "=" not in overrides_list
+                and overrides_list[0].endswith(".toml")
+            ):
+                with Path(overrides_list[0]).open("rb") as config_file:
+                    overrides = flatten_dict(tomli.load(config_file))
+            else:
+                toml_str = "\n".join(overrides_list)
+                overrides.update(tomli.loads(toml_str))
+
+    return overrides

flwr/common/constant.py

@@ -57,6 +57,9 @@ APP_DIR = "apps"
 FAB_CONFIG_FILE = "pyproject.toml"
 FLWR_HOME = "FLWR_HOME"
 
+# Constants entries in Node config for Simulation
+PARTITION_ID_KEY = "partition-id"
+NUM_PARTITIONS_KEY = "num-partitions"
 
 GRPC_ADAPTER_METADATA_FLOWER_VERSION_KEY = "flower-version"
 GRPC_ADAPTER_METADATA_SHOULD_EXIT_KEY = "should-exit"

flwr/common/context.py

@@ -16,9 +16,9 @@
 
 
 from dataclasses import dataclass
-from typing import Optional
 
 from .record import RecordSet
+from .typing import UserConfig
 
 
 @dataclass
@@ -27,6 +27,11 @@ class Context:
 
     Parameters
     ----------
+    node_id : int
+        The ID that identifies the node.
+    node_config : UserConfig
+        A config (key/value mapping) unique to the node and independent of the
+        `run_config`. This config persists across all runs this node participates in.
     state : RecordSet
         Holds records added by the entity in a given run and that will stay local.
         This means that the data it holds will never leave the system it's running from.
@@ -34,15 +39,25 @@ class Context:
         executing mods. It can also be used as a memory to access
         at different points during the lifecycle of this entity (e.g. across
         multiple rounds)
-    partition_id : Optional[int] (default: None)
-        An index that specifies the data partition that the ClientApp using this Context
-        object should make use of. Setting this attribute is better suited for
-        simulation or proto typing setups.
+    run_config : UserConfig
+        A config (key/value mapping) held by the entity in a given run and that will
+        stay local. It can be used at any point during the lifecycle of this entity
+        (e.g. across multiple rounds)
     """
 
+    node_id: int
+    node_config: UserConfig
     state: RecordSet
-    partition_id: Optional[int]
-
-    def __init__(self, state: RecordSet, partition_id: Optional[int] = None) -> None:
+    run_config: UserConfig
+
+    def __init__(  # pylint: disable=too-many-arguments
+        self,
+        node_id: int,
+        node_config: UserConfig,
+        state: RecordSet,
+        run_config: UserConfig,
+    ) -> None:
+        self.node_id = node_id
+        self.node_config = node_config
         self.state = state
-        self.partition_id = partition_id
+        self.run_config = run_config

flwr/common/logger.py

@@ -197,6 +197,31 @@ def warn_deprecated_feature(name: str) -> None:
     )
 
 
+def warn_deprecated_feature_with_example(
+    deprecation_message: str, example_message: str, code_example: str
+) -> None:
+    """Warn if a feature is deprecated and show code example."""
+    log(
+        WARN,
+        """DEPRECATED FEATURE: %s
+
+            Check the following `FEATURE UPDATE` warning message for the preferred
+            new mechanism to use this feature in Flower.
+        """,
+        deprecation_message,
+    )
+    log(
+        WARN,
+        """FEATURE UPDATE: %s
+        ------------------------------------------------------------
+        %s
+        ------------------------------------------------------------
+        """,
+        example_message,
+        code_example,
+    )
+
+
 def warn_unsupported_feature(name: str) -> None:
     """Warn the user when they use an unsupported feature."""
     log(

flwr/common/serde.py

@@ -671,3 +671,48 @@ def message_from_taskres(taskres: TaskRes) -> Message:
     )
     message.metadata.created_at = taskres.task.created_at
     return message
+
+
+# === User configs ===
+
+
+def user_config_to_proto(user_config: typing.UserConfig) -> Any:
+    """Serialize `UserConfig` to ProtoBuf."""
+    proto = {}
+    for key, value in user_config.items():
+        proto[key] = user_config_value_to_proto(value)
+    return proto
+
+
+def user_config_from_proto(proto: Any) -> typing.UserConfig:
+    """Deserialize `UserConfig` from ProtoBuf."""
+    metrics = {}
+    for key, value in proto.items():
+        metrics[key] = user_config_value_from_proto(value)
+    return metrics
+
+
+def user_config_value_to_proto(user_config_value: typing.UserConfigValue) -> Scalar:
+    """Serialize `UserConfigValue` to ProtoBuf."""
+    if isinstance(user_config_value, bool):
+        return Scalar(bool=user_config_value)
+
+    if isinstance(user_config_value, float):
+        return Scalar(double=user_config_value)
+
+    if isinstance(user_config_value, int):
+        return Scalar(sint64=user_config_value)
+
+    if isinstance(user_config_value, str):
+        return Scalar(string=user_config_value)
+
+    raise ValueError(
+        f"Accepted types: {bool, float, int, str} (but not {type(user_config_value)})"
+    )
+
+
+def user_config_value_from_proto(scalar_msg: Scalar) -> typing.UserConfigValue:
+    """Deserialize `UserConfigValue` from ProtoBuf."""
+    scalar_field = scalar_msg.WhichOneof("scalar")
+    scalar = getattr(scalar_msg, cast(str, scalar_field))
+    return cast(typing.UserConfigValue, scalar)

flwr/common/telemetry.py

@@ -64,6 +64,18 @@ def _get_home() -> Path:
     return Path().home()
 
 
+def _get_partner_id() -> str:
+    """Get partner ID."""
+    partner_id = os.getenv("FLWR_TELEMETRY_PARTNER_ID")
+    if not partner_id:
+        return "unavailable"
+    try:
+        uuid.UUID(partner_id)
+    except ValueError:
+        partner_id = "invalid"
+    return partner_id
+
+
 def _get_source_id() -> str:
     """Get existing or new source ID."""
     source_id = "unavailable"
@@ -177,6 +189,7 @@ state: Dict[str, Union[Optional[str], Optional[ThreadPoolExecutor]]] = {
     "executor": None,
     "source": None,
     "cluster": None,
+    "partner": None,
 }
 
 
@@ -202,11 +215,15 @@ def create_event(event_type: EventType, event_details: Optional[Dict[str, Any]])
     if state["cluster"] is None:
         state["cluster"] = str(uuid.uuid4())
 
+    if state["partner"] is None:
+        state["partner"] = _get_partner_id()
+
     if event_details is None:
         event_details = {}
 
     date = datetime.datetime.now(tz=datetime.timezone.utc).isoformat()
     context = {
+        "partner": state["partner"],
         "source": state["source"],
         "cluster": state["cluster"],
         "date": date,