flwr-nightly 1.13.0.dev20241021__py3-none-any.whl → 1.13.0.dev20241111__py3-none-any.whl

flwr/cli/build.py

@@ -81,8 +81,8 @@ def build(
     if not is_valid_project_name(app.name):
         typer.secho(
             f"❌ The project name {app.name} is invalid, "
-            "a valid project name must start with a letter or an underscore, "
-            "and can only contain letters, digits, and underscores.",
+            "a valid project name must start with a letter, "
+            "and can only contain letters, digits, and hyphens.",
             fg=typer.colors.RED,
             bold=True,
         )

flwr/cli/config_utils.py

@@ -20,6 +20,7 @@ from pathlib import Path
 from typing import IO, Any, Optional, Union, get_args
 
 import tomli
+import typer
 
 from flwr.common import object_ref
 from flwr.common.typing import UserConfigValue
@@ -227,3 +228,99 @@ def load_from_string(toml_content: str) -> Optional[dict[str, Any]]:
         return data
     except tomli.TOMLDecodeError:
         return None
+
+
+def validate_project_config(
+    config: Union[dict[str, Any], None], errors: list[str], warnings: list[str]
+) -> dict[str, Any]:
+    """Validate and return the Flower project configuration."""
+    if config is None:
+        typer.secho(
+            "Project configuration could not be loaded.\n"
+            "pyproject.toml is invalid:\n"
+            + "\n".join([f"- {line}" for line in errors]),
+            fg=typer.colors.RED,
+            bold=True,
+        )
+        raise typer.Exit(code=1)
+
+    if warnings:
+        typer.secho(
+            "Project configuration is missing the following "
+            "recommended properties:\n" + "\n".join([f"- {line}" for line in warnings]),
+            fg=typer.colors.RED,
+            bold=True,
+        )
+
+    typer.secho("Success", fg=typer.colors.GREEN)
+
+    return config
+
+
+def validate_federation_in_project_config(
+    federation: Optional[str], config: dict[str, Any]
+) -> tuple[str, dict[str, Any]]:
+    """Validate the federation name in the Flower project configuration."""
+    federation = federation or config["tool"]["flwr"]["federations"].get("default")
+
+    if federation is None:
+        typer.secho(
+            "❌ No federation name was provided and the project's `pyproject.toml` "
+            "doesn't declare a default federation (with an Exec API address or an "
+            "`options.num-supernodes` value).",
+            fg=typer.colors.RED,
+            bold=True,
+        )
+        raise typer.Exit(code=1)
+
+    # Validate the federation exists in the configuration
+    federation_config = config["tool"]["flwr"]["federations"].get(federation)
+    if federation_config is None:
+        available_feds = {
+            fed for fed in config["tool"]["flwr"]["federations"] if fed != "default"
+        }
+        typer.secho(
+            f"❌ There is no `{federation}` federation declared in the "
+            "`pyproject.toml`.\n The following federations were found:\n\n"
+            + "\n".join(available_feds),
+            fg=typer.colors.RED,
+            bold=True,
+        )
+        raise typer.Exit(code=1)
+
+    return federation, federation_config
+
+
+def validate_certificate_in_federation_config(
+    app: Path, federation_config: dict[str, Any]
+) -> tuple[bool, Optional[bytes]]:
+    """Validate the certificates in the Flower project configuration."""
+    insecure_str = federation_config.get("insecure")
+    if root_certificates := federation_config.get("root-certificates"):
+        root_certificates_bytes = (app / root_certificates).read_bytes()
+        if insecure := bool(insecure_str):
+            typer.secho(
+                "❌ `root_certificates` were provided but the `insecure` parameter "
+                "is set to `True`.",
+                fg=typer.colors.RED,
+                bold=True,
+            )
+            raise typer.Exit(code=1)
+    else:
+        root_certificates_bytes = None
+        if insecure_str is None:
+            typer.secho(
+                "❌ To disable TLS, set `insecure = true` in `pyproject.toml`.",
+                fg=typer.colors.RED,
+                bold=True,
+            )
+            raise typer.Exit(code=1)
+        if not (insecure := bool(insecure_str)):
+            typer.secho(
+                "❌ No certificate were given yet `insecure` is set to `False`.",
+                fg=typer.colors.RED,
+                bold=True,
+            )
+            raise typer.Exit(code=1)
+
+    return insecure, root_certificates_bytes

flwr/cli/log.py

@@ -14,33 +14,38 @@
 # ==============================================================================
 """Flower command line interface `log` command."""
 
-import sys
 import time
 from logging import DEBUG, ERROR, INFO
 from pathlib import Path
-from typing import Annotated, Optional
+from typing import Annotated, Any, Optional, cast
 
 import grpc
 import typer
 
-from flwr.cli.config_utils import load_and_validate
+from flwr.cli.config_utils import (
+    load_and_validate,
+    validate_certificate_in_federation_config,
+    validate_federation_in_project_config,
+    validate_project_config,
+)
+from flwr.common.constant import CONN_RECONNECT_INTERVAL, CONN_REFRESH_PERIOD
 from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel
 from flwr.common.logger import log as logger
 from flwr.proto.exec_pb2 import StreamLogsRequest  # pylint: disable=E0611
 from flwr.proto.exec_pb2_grpc import ExecStub
 
-CONN_REFRESH_PERIOD = 60  # Connection refresh period for log streaming (seconds)
-
 
 def start_stream(
     run_id: int, channel: grpc.Channel, refresh_period: int = CONN_REFRESH_PERIOD
 ) -> None:
     """Start log streaming for a given run ID."""
+    stub = ExecStub(channel)
+    after_timestamp = 0.0
     try:
+        logger(INFO, "Starting logstream for run_id `%s`", run_id)
         while True:
-            logger(INFO, "Starting logstream for run_id `%s`", run_id)
-            stream_logs(run_id, channel, refresh_period)
-            time.sleep(2)
+            after_timestamp = stream_logs(run_id, stub, refresh_period, after_timestamp)
+            time.sleep(CONN_RECONNECT_INTERVAL)
             logger(DEBUG, "Reconnecting to logstream")
     except KeyboardInterrupt:
         logger(INFO, "Exiting logstream")
@@ -54,16 +59,44 @@ def start_stream(
         channel.close()
 
 
-def stream_logs(run_id: int, channel: grpc.Channel, duration: int) -> None:
-    """Stream logs from the beginning of a run with connection refresh."""
-    start_time = time.time()
-    stub = ExecStub(channel)
-    req = StreamLogsRequest(run_id=run_id)
+def stream_logs(
+    run_id: int, stub: ExecStub, duration: int, after_timestamp: float
+) -> float:
+    """Stream logs from the beginning of a run with connection refresh.
+
+    Parameters
+    ----------
+    run_id : int
+        The identifier of the run.
+    stub : ExecStub
+        The gRPC stub to interact with the Exec service.
+    duration : int
+        The timeout duration for each stream connection in seconds.
+    after_timestamp : float
+        The timestamp to start streaming logs from.
+
+    Returns
+    -------
+    float
+        The latest timestamp from the streamed logs or the provided `after_timestamp`
+        if no logs are returned.
+    """
+    req = StreamLogsRequest(run_id=run_id, after_timestamp=after_timestamp)
+
+    latest_timestamp = 0.0
+    res = None
+    try:
+        for res in stub.StreamLogs(req, timeout=duration):
+            print(res.log_output, end="")
+    except grpc.RpcError as e:
+        # pylint: disable=E1101
+        if e.code() != grpc.StatusCode.DEADLINE_EXCEEDED:
+            raise e
+    finally:
+        if res is not None:
+            latest_timestamp = cast(float, res.latest_timestamp)
 
-    for res in stub.StreamLogs(req):
-        print(res.log_output)
-        if time.time() - start_time > duration:
-            break
+    return max(latest_timestamp, after_timestamp)
 
 
 def print_logs(run_id: int, channel: grpc.Channel, timeout: int) -> None:
@@ -124,100 +157,33 @@ def log(
 
     pyproject_path = app / "pyproject.toml" if app else None
     config, errors, warnings = load_and_validate(path=pyproject_path)
-
-    if config is None:
-        typer.secho(
-            "Project configuration could not be loaded.\n"
-            "pyproject.toml is invalid:\n"
-            + "\n".join([f"- {line}" for line in errors]),
-            fg=typer.colors.RED,
-            bold=True,
-        )
-        sys.exit()
-
-    if warnings:
-        typer.secho(
-            "Project configuration is missing the following "
-            "recommended properties:\n" + "\n".join([f"- {line}" for line in warnings]),
-            fg=typer.colors.RED,
-            bold=True,
-        )
-
-    typer.secho("Success", fg=typer.colors.GREEN)
-
-    federation = federation or config["tool"]["flwr"]["federations"].get("default")
-
-    if federation is None:
-        typer.secho(
-            "❌ No federation name was provided and the project's `pyproject.toml` "
-            "doesn't declare a default federation (with a SuperExec address or an "
-            "`options.num-supernodes` value).",
-            fg=typer.colors.RED,
-            bold=True,
-        )
-        raise typer.Exit(code=1)
-
-    # Validate the federation exists in the configuration
-    federation_config = config["tool"]["flwr"]["federations"].get(federation)
-    if federation_config is None:
-        available_feds = {
-            fed for fed in config["tool"]["flwr"]["federations"] if fed != "default"
-        }
-        typer.secho(
-            f"❌ There is no `{federation}` federation declared in the "
-            "`pyproject.toml`.\n The following federations were found:\n\n"
-            + "\n".join(available_feds),
-            fg=typer.colors.RED,
-            bold=True,
-        )
-        raise typer.Exit(code=1)
+    config = validate_project_config(config, errors, warnings)
+    federation, federation_config = validate_federation_in_project_config(
+        federation, config
+    )
 
     if "address" not in federation_config:
         typer.secho(
-            "❌ `flwr log` currently works with `SuperExec`. Ensure that the correct"
-            "`SuperExec` address is provided in the `pyproject.toml`.",
+            "❌ `flwr log` currently works with Exec API. Ensure that the correct"
+            "Exec API address is provided in the `pyproject.toml`.",
             fg=typer.colors.RED,
             bold=True,
         )
         raise typer.Exit(code=1)
 
-    _log_with_superexec(federation_config, run_id, stream)
+    _log_with_exec_api(app, federation_config, run_id, stream)
 
 
-# pylint: disable-next=too-many-branches
-def _log_with_superexec(
-    federation_config: dict[str, str],
+def _log_with_exec_api(
+    app: Path,
+    federation_config: dict[str, Any],
     run_id: int,
     stream: bool,
 ) -> None:
-    insecure_str = federation_config.get("insecure")
-    if root_certificates := federation_config.get("root-certificates"):
-        root_certificates_bytes = Path(root_certificates).read_bytes()
-        if insecure := bool(insecure_str):
-            typer.secho(
-                "❌ `root_certificates` were provided but the `insecure` parameter"
-                "is set to `True`.",
-                fg=typer.colors.RED,
-                bold=True,
-            )
-            raise typer.Exit(code=1)
-    else:
-        root_certificates_bytes = None
-        if insecure_str is None:
-            typer.secho(
-                "❌ To disable TLS, set `insecure = true` in `pyproject.toml`.",
-                fg=typer.colors.RED,
-                bold=True,
-            )
-            raise typer.Exit(code=1)
-        if not (insecure := bool(insecure_str)):
-            typer.secho(
-                "❌ No certificate were given yet `insecure` is set to `False`.",
-                fg=typer.colors.RED,
-                bold=True,
-            )
-            raise typer.Exit(code=1)
 
+    insecure, root_certificates_bytes = validate_certificate_in_federation_config(
+        app, federation_config
+    )
     channel = create_channel(
         server_address=federation_config["address"],
         insecure=insecure,

flwr/cli/new/templates/app/code/flwr_tune/dataset.py.tpl

@@ -71,7 +71,7 @@ def load_data(partition_id: int, num_partitions: int, dataset_name: str):
             partitioners={"train": partitioner},
         )
     client_trainset = FDS.load_partition(partition_id, "train")
-    client_trainset = reformat(client_trainset, llm_task="generalnlp")
+    client_trainset = reformat(client_trainset, llm_task="$llm_challenge_str")
     return client_trainset
 
 

flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl

@@ -10,6 +10,7 @@ license = "Apache-2.0"
 dependencies = [
     "flwr[simulation]>=1.12.0",
     "flwr-datasets>=0.3.0",
+    "torch==2.3.1",
     "trl==0.8.1",
     "bitsandbytes==0.43.0",
     "scipy==1.13.0",

flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl

@@ -10,7 +10,7 @@ license = "Apache-2.0"
 dependencies = [
     "flwr[simulation]>=1.12.0",
     "flwr-datasets[vision]>=0.3.0",
-    "tensorflow>=2.11.1",
+    "tensorflow>=2.11.1,<2.18.0",
 ]
 
 [tool.hatch.build.targets.wheel]

flwr/cli/run/run.py

@@ -16,7 +16,6 @@
 
 import json
 import subprocess
-import sys
 from logging import DEBUG
 from pathlib import Path
 from typing import Annotated, Any, Optional
@@ -24,11 +23,24 @@ from typing import Annotated, Any, Optional
 import typer
 
 from flwr.cli.build import build
-from flwr.cli.config_utils import load_and_validate
-from flwr.common.config import flatten_dict, parse_config_args
+from flwr.cli.config_utils import (
+    load_and_validate,
+    validate_certificate_in_federation_config,
+    validate_federation_in_project_config,
+    validate_project_config,
+)
+from flwr.common.config import (
+    flatten_dict,
+    parse_config_args,
+    user_config_to_configsrecord,
+)
 from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel
 from flwr.common.logger import log
-from flwr.common.serde import fab_to_proto, user_config_to_proto
+from flwr.common.serde import (
+    configs_record_to_proto,
+    fab_to_proto,
+    user_config_to_proto,
+)
 from flwr.common.typing import Fab
 from flwr.proto.exec_pb2 import StartRunRequest  # pylint: disable=E0611
 from flwr.proto.exec_pb2_grpc import ExecStub
@@ -79,96 +91,28 @@ def run(
 
     pyproject_path = app / "pyproject.toml" if app else None
     config, errors, warnings = load_and_validate(path=pyproject_path)
-
-    if config is None:
-        typer.secho(
-            "Project configuration could not be loaded.\n"
-            "pyproject.toml is invalid:\n"
-            + "\n".join([f"- {line}" for line in errors]),
-            fg=typer.colors.RED,
-            bold=True,
-        )
-        sys.exit()
-
-    if warnings:
-        typer.secho(
-            "Project configuration is missing the following "
-            "recommended properties:\n" + "\n".join([f"- {line}" for line in warnings]),
-            fg=typer.colors.RED,
-            bold=True,
-        )
-
-    typer.secho("Success", fg=typer.colors.GREEN)
-
-    federation = federation or config["tool"]["flwr"]["federations"].get("default")
-
-    if federation is None:
-        typer.secho(
-            "❌ No federation name was provided and the project's `pyproject.toml` "
-            "doesn't declare a default federation (with a SuperExec address or an "
-            "`options.num-supernodes` value).",
-            fg=typer.colors.RED,
-            bold=True,
-        )
-        raise typer.Exit(code=1)
-
-    # Validate the federation exists in the configuration
-    federation_config = config["tool"]["flwr"]["federations"].get(federation)
-    if federation_config is None:
-        available_feds = {
-            fed for fed in config["tool"]["flwr"]["federations"] if fed != "default"
-        }
-        typer.secho(
-            f"❌ There is no `{federation}` federation declared in "
-            "`pyproject.toml`.\n The following federations were found:\n\n"
-            + "\n".join(available_feds),
-            fg=typer.colors.RED,
-            bold=True,
-        )
-        raise typer.Exit(code=1)
+    config = validate_project_config(config, errors, warnings)
+    federation, federation_config = validate_federation_in_project_config(
+        federation, config
+    )
 
     if "address" in federation_config:
-        _run_with_superexec(app, federation_config, config_overrides, stream)
+        _run_with_exec_api(app, federation_config, config_overrides, stream)
     else:
-        _run_without_superexec(app, federation_config, config_overrides, federation)
+        _run_without_exec_api(app, federation_config, config_overrides, federation)
 
 
-# pylint: disable=too-many-locals
-def _run_with_superexec(
+# pylint: disable-next=too-many-locals
+def _run_with_exec_api(
     app: Path,
     federation_config: dict[str, Any],
     config_overrides: Optional[list[str]],
     stream: bool,
 ) -> None:
 
-    insecure_str = federation_config.get("insecure")
-    if root_certificates := federation_config.get("root-certificates"):
-        root_certificates_bytes = (app / root_certificates).read_bytes()
-        if insecure := bool(insecure_str):
-            typer.secho(
-                "❌ `root_certificates` were provided but the `insecure` parameter"
-                "is set to `True`.",
-                fg=typer.colors.RED,
-                bold=True,
-            )
-            raise typer.Exit(code=1)
-    else:
-        root_certificates_bytes = None
-        if insecure_str is None:
-            typer.secho(
-                "❌ To disable TLS, set `insecure = true` in `pyproject.toml`.",
-                fg=typer.colors.RED,
-                bold=True,
-            )
-            raise typer.Exit(code=1)
-        if not (insecure := bool(insecure_str)):
-            typer.secho(
-                "❌ No certificate were given yet `insecure` is set to `False`.",
-                fg=typer.colors.RED,
-                bold=True,
-            )
-            raise typer.Exit(code=1)
-
+    insecure, root_certificates_bytes = validate_certificate_in_federation_config(
+        app, federation_config
+    )
     channel = create_channel(
         server_address=federation_config["address"],
         insecure=insecure,
@@ -183,16 +127,18 @@ def _run_with_superexec(
     content = Path(fab_path).read_bytes()
     fab = Fab(fab_hash, content)
 
+    # Construct a `ConfigsRecord` out of a flattened `UserConfig`
+    fed_conf = flatten_dict(federation_config.get("options", {}))
+    c_record = user_config_to_configsrecord(fed_conf)
+
     req = StartRunRequest(
         fab=fab_to_proto(fab),
         override_config=user_config_to_proto(parse_config_args(config_overrides)),
-        federation_config=user_config_to_proto(
-            flatten_dict(federation_config.get("options"))
-        ),
+        federation_options=configs_record_to_proto(c_record),
     )
     res = stub.StartRun(req)
 
-    # Delete FAB file once it has been sent to the SuperExec
+    # Delete FAB file once it has been sent to the Exec API
     Path(fab_path).unlink()
     typer.secho(f"🎊 Successfully started run {res.run_id}", fg=typer.colors.GREEN)
 
@@ -200,7 +146,7 @@ def _run_with_superexec(
         start_stream(res.run_id, channel, CONN_REFRESH_PERIOD)
 
 
-def _run_without_superexec(
+def _run_without_exec_api(
     app: Optional[Path],
     federation_config: dict[str, Any],
     config_overrides: Optional[list[str]],

flwr/client/app.py

@@ -32,11 +32,14 @@ from flwr.cli.config_utils import get_fab_metadata
 from flwr.cli.install import install_from_fab
 from flwr.client.client import Client
 from flwr.client.client_app import ClientApp, LoadClientAppError
+from flwr.client.nodestate.nodestate_factory import NodeStateFactory
 from flwr.client.typing import ClientFnExt
 from flwr.common import GRPC_MAX_MESSAGE_LENGTH, Context, EventType, Message, event
 from flwr.common.address import parse_address
 from flwr.common.constant import (
     CLIENTAPPIO_API_DEFAULT_ADDRESS,
+    ISOLATION_MODE_PROCESS,
+    ISOLATION_MODE_SUBPROCESS,
     MISSING_EXTRA_REST,
     RUN_ID_NUM_BYTES,
     TRANSPORT_TYPE_GRPC_ADAPTER,
@@ -52,18 +55,15 @@ from flwr.common.retry_invoker import RetryInvoker, RetryState, exponential
 from flwr.common.typing import Fab, Run, UserConfig
 from flwr.proto.clientappio_pb2_grpc import add_ClientAppIoServicer_to_server
 from flwr.server.superlink.fleet.grpc_bidi.grpc_server import generic_create_grpc_server
-from flwr.server.superlink.state.utils import generate_rand_int_from_bytes
+from flwr.server.superlink.linkstate.utils import generate_rand_int_from_bytes
 
 from .clientapp.clientappio_servicer import ClientAppInputs, ClientAppIoServicer
 from .grpc_adapter_client.connection import grpc_adapter
 from .grpc_client.connection import grpc_connection
 from .grpc_rere_client.connection import grpc_request_response
 from .message_handler.message_handler import handle_control_message
-from .node_state import NodeState
 from .numpy_client import NumPyClient
-
-ISOLATION_MODE_SUBPROCESS = "subprocess"
-ISOLATION_MODE_PROCESS = "process"
+from .run_info_store import DeprecatedRunInfoStore
 
 
 def _check_actionable_client(
@@ -364,8 +364,10 @@ def start_client_internal(
         on_backoff=_on_backoff,
     )
 
-    # NodeState gets initialized when the first connection is established
-    node_state: Optional[NodeState] = None
+    # DeprecatedRunInfoStore gets initialized when the first connection is established
+    run_info_store: Optional[DeprecatedRunInfoStore] = None
+    state_factory = NodeStateFactory()
+    state = state_factory.state()
 
     runs: dict[int, Run] = {}
 
@@ -382,7 +384,7 @@ def start_client_internal(
             receive, send, create_node, delete_node, get_run, get_fab = conn
 
             # Register node when connecting the first time
-            if node_state is None:
+            if run_info_store is None:
                 if create_node is None:
                     if transport not in ["grpc-bidi", None]:
                         raise NotImplementedError(
@@ -391,19 +393,20 @@ def start_client_internal(
                         )
                     # gRPC-bidi doesn't have the concept of node_id,
                     # so we set it to -1
-                    node_state = NodeState(
+                    run_info_store = DeprecatedRunInfoStore(
                         node_id=-1,
                         node_config={},
                     )
                 else:
                     # Call create_node fn to register node
-                    node_id: Optional[int] = (  # pylint: disable=assignment-from-none
-                        create_node()
-                    )  # pylint: disable=not-callable
-                    if node_id is None:
-                        raise ValueError("Node registration failed")
-                    node_state = NodeState(
-                        node_id=node_id,
+                    # and store node_id in state
+                    if (node_id := create_node()) is None:
+                        raise ValueError(
+                            "Failed to register SuperNode with the SuperLink"
+                        )
+                    state.set_node_id(node_id)
+                    run_info_store = DeprecatedRunInfoStore(
+                        node_id=state.get_node_id(),
                         node_config=node_config,
                     )
 
@@ -461,7 +464,7 @@ def start_client_internal(
                     run.fab_id, run.fab_version = fab_id, fab_version
 
                     # Register context for this run
-                    node_state.register_context(
+                    run_info_store.register_context(
                         run_id=run_id,
                         run=run,
                         flwr_path=flwr_path,
@@ -469,7 +472,7 @@ def start_client_internal(
                     )
 
                     # Retrieve context for this run
-                    context = node_state.retrieve_context(run_id=run_id)
+                    context = run_info_store.retrieve_context(run_id=run_id)
                     # Create an error reply message that will never be used to prevent
                     # the used-before-assignment linting error
                     reply_message = message.create_error_reply(
@@ -542,7 +545,7 @@ def start_client_internal(
                             # Raise exception, crash process
                             raise ex
 
-                        # Don't update/change NodeState
+                        # Don't update/change DeprecatedRunInfoStore
 
                         e_code = ErrorCode.CLIENT_APP_RAISED_EXCEPTION
                         # Ex fmt: "<class 'ZeroDivisionError'>:<'division by zero'>"
@@ -567,7 +570,7 @@ def start_client_internal(
                         )
                     else:
                         # No exception, update node state
-                        node_state.update_context(
+                        run_info_store.update_context(
                             run_id=run_id,
                             context=context,
                         )