flwr-nightly 1.10.0.dev20240624__tar.gz → 1.10.0.dev20240629__tar.gz

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.10.0.dev20240624
+Version: 1.10.0.dev20240629
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "flwr-nightly"
-version = "1.10.0.dev20240624"
+version = "1.10.0.dev20240629"
 description = "Flower: A Friendly Federated Learning Framework"
 license = "Apache-2.0"
 authors = ["The Flower Authors <hello@flower.ai>"]
@@ -122,7 +122,7 @@ mdformat-gfm = "==0.3.5"
 mdformat-frontmatter = "==2.0.1"
 mdformat-beautysh = "==0.1.1"
 mdformat-myst = "==0.1.5"
-twine = "==4.0.2"
+twine = "==5.1.1"
 pyroma = "==4.2"
 check-wheel-contents = "==0.4.0"
 GitPython = "==3.1.32"

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/src/py/flwr/cli/build.py

@@ -31,7 +31,7 @@ from .utils import get_sha256_hash, is_valid_project_name
 def build(
     directory: Annotated[
         Optional[Path],
-        typer.Option(help="The Flower project directory to bundle into a FAB"),
+        typer.Option(help="Path of the Flower project to bundle into a FAB"),
     ] = None,
 ) -> str:
     """Build a Flower project into a Flower App Bundle (FAB).
@@ -118,7 +118,7 @@ def build(
         fab_file.writestr(".info/CONTENT", list_file_content)
 
     typer.secho(
-        f"🎊 Successfully built {fab_filename}.", fg=typer.colors.GREEN, bold=True
+        f"🎊 Successfully built {fab_filename}", fg=typer.colors.GREEN, bold=True
     )
 
     return fab_filename

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/src/py/flwr/cli/run/run.py

@@ -17,12 +17,14 @@
 import sys
 from enum import Enum
 from logging import DEBUG
+from pathlib import Path
 from typing import Optional
 
 import typer
 from typing_extensions import Annotated
 
 from flwr.cli import config_utils
+from flwr.cli.build import build
 from flwr.common.constant import SUPEREXEC_DEFAULT_ADDRESS
 from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel
 from flwr.common.logger import log
@@ -52,10 +54,14 @@ def run(
             case_sensitive=False, help="Use this flag to use the new SuperExec API"
         ),
     ] = False,
+    directory: Annotated[
+        Optional[Path],
+        typer.Option(help="Path of the Flower project to run"),
+    ] = None,
 ) -> None:
     """Run Flower project."""
     if use_superexec:
-        _start_superexec_run()
+        _start_superexec_run(directory)
         return
 
     typer.secho("Loading project configuration... ", fg=typer.colors.BLUE)
@@ -109,7 +115,7 @@ def run(
         )
 
 
-def _start_superexec_run() -> None:
+def _start_superexec_run(directory: Optional[Path]) -> None:
     def on_channel_state_change(channel_connectivity: str) -> None:
         """Log channel connectivity."""
         log(DEBUG, channel_connectivity)
@@ -124,5 +130,8 @@ def _start_superexec_run() -> None:
     channel.subscribe(on_channel_state_change)
     stub = ExecStub(channel)
 
-    req = StartRunRequest()
-    stub.StartRun(req)
+    fab_path = build(directory)
+
+    req = StartRunRequest(fab_file=Path(fab_path).read_bytes())
+    res = stub.StartRun(req)
+    typer.secho(f"🎊 Successfully started run {res.run_id}", fg=typer.colors.GREEN)

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/src/py/flwr/client/app.py

@@ -191,6 +191,7 @@ def _start_client_internal(
     ] = None,
     max_retries: Optional[int] = None,
     max_wait_time: Optional[float] = None,
+    partition_id: Optional[int] = None,
 ) -> None:
     """Start a Flower client node which connects to a Flower server.
 
@@ -234,6 +235,9 @@ def _start_client_internal(
         The maximum duration before the client stops trying to
         connect to the server in case of connection error.
         If set to None, there is no limit to the total time.
+    partitioni_id: Optional[int] (default: None)
+        The data partition index associated with this node. Better suited for
+        prototyping purposes.
     """
     if insecure is None:
         insecure = root_certificates is None
@@ -309,7 +313,7 @@ def _start_client_internal(
         on_backoff=_on_backoff,
     )
 
-    node_state = NodeState()
+    node_state = NodeState(partition_id=partition_id)
     # run_id -> (fab_id, fab_version)
     run_info: Dict[int, Tuple[str, str]] = {}
 

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/src/py/flwr/client/message_handler/message_handler.py

@@ -93,7 +93,7 @@ def handle_legacy_message_from_msgtype(
     client_fn: ClientFn, message: Message, context: Context
 ) -> Message:
     """Handle legacy message in the inner most mod."""
-    client = client_fn(str(message.metadata.partition_id))
+    client = client_fn(str(context.partition_id))
 
     # Check if NumPyClient is returend
     if isinstance(client, NumPyClient):

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/src/py/flwr/client/node_state.py

@@ -15,7 +15,7 @@
 """Node state."""
 
 
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 from flwr.common import Context, RecordSet
 
@@ -23,14 +23,17 @@ from flwr.common import Context, RecordSet
 class NodeState:
     """State of a node where client nodes execute runs."""
 
-    def __init__(self) -> None:
+    def __init__(self, partition_id: Optional[int]) -> None:
         self._meta: Dict[str, Any] = {}  # holds metadata about the node
         self.run_contexts: Dict[int, Context] = {}
+        self._partition_id = partition_id
 
     def register_context(self, run_id: int) -> None:
         """Register new run context for this node."""
         if run_id not in self.run_contexts:
-            self.run_contexts[run_id] = Context(state=RecordSet())
+            self.run_contexts[run_id] = Context(
+                state=RecordSet(), partition_id=self._partition_id
+            )
 
     def retrieve_context(self, run_id: int) -> Context:
         """Get run context given a run_id."""

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/src/py/flwr/client/node_state_tests.py

@@ -41,7 +41,7 @@ def test_multirun_in_node_state() -> None:
     expected_values = {0: "1", 1: "1" * 3, 2: "1" * 2, 3: "1", 5: "1"}
 
     # NodeState
-    node_state = NodeState()
+    node_state = NodeState(partition_id=None)
 
     for task in tasks:
         run_id = task.run_id

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/src/py/flwr/client/supernode/app.py

@@ -67,6 +67,7 @@ def run_supernode() -> None:
         authentication_keys=authentication_keys,
         max_retries=args.max_retries,
         max_wait_time=args.max_wait_time,
+        partition_id=args.partition_id,
     )
 
     # Graceful shutdown
@@ -373,6 +374,13 @@ def _parse_args_common(parser: argparse.ArgumentParser) -> None:
         type=str,
         help="The SuperNode's public key (as a path str) to enable authentication.",
     )
+    parser.add_argument(
+        "--partition-id",
+        type=int,
+        help="The data partition index associated with this SuperNode. Better suited "
+        "for prototyping purposes where a SuperNode might only load a fraction of an "
+        "artificially partitioned dataset (e.g. using `flwr-datasets`)",
+    )
 
 
 def _try_setup_client_authentication(

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/src/py/flwr/common/constant.py

@@ -46,6 +46,9 @@ PING_BASE_MULTIPLIER = 0.8
 PING_RANDOM_RANGE = (-0.1, 0.1)
 PING_MAX_INTERVAL = 1e300
 
+# IDs
+RUN_ID_NUM_BYTES = 8
+NODE_ID_NUM_BYTES = 8
 GRPC_ADAPTER_METADATA_FLOWER_VERSION_KEY = "flower-version"
 GRPC_ADAPTER_METADATA_SHOULD_EXIT_KEY = "should-exit"
 

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/src/py/flwr/common/context.py

@@ -16,13 +16,14 @@
 
 
 from dataclasses import dataclass
+from typing import Optional
 
 from .record import RecordSet
 
 
 @dataclass
 class Context:
-    """State of your run.
+    """Context of your run.
 
     Parameters
     ----------
@@ -33,6 +34,15 @@ class Context:
         executing mods. It can also be used as a memory to access
         at different points during the lifecycle of this entity (e.g. across
         multiple rounds)
+    partition_id : Optional[int] (default: None)
+        An index that specifies the data partition that the ClientApp using this Context
+        object should make use of. Setting this attribute is better suited for
+        simulation or proto typing setups.
     """
 
     state: RecordSet
+    partition_id: Optional[int]
+
+    def __init__(self, state: RecordSet, partition_id: Optional[int] = None) -> None:
+        self.state = state
+        self.partition_id = partition_id

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/src/py/flwr/common/message.py

@@ -48,10 +48,6 @@ class Metadata:  # pylint: disable=too-many-instance-attributes
     message_type : str
         A string that encodes the action to be executed on
         the receiving end.
-    partition_id : Optional[int]
-        An identifier that can be used when loading a particular
-        data partition for a ClientApp. Making use of this identifier
-        is more relevant when conducting simulations.
     """
 
     def __init__(  # pylint: disable=too-many-arguments
@@ -64,7 +60,6 @@ class Metadata:  # pylint: disable=too-many-instance-attributes
         group_id: str,
         ttl: float,
         message_type: str,
-        partition_id: int | None = None,
     ) -> None:
         var_dict = {
             "_run_id": run_id,
@@ -75,7 +70,6 @@ class Metadata:  # pylint: disable=too-many-instance-attributes
             "_group_id": group_id,
             "_ttl": ttl,
             "_message_type": message_type,
-            "_partition_id": partition_id,
         }
         self.__dict__.update(var_dict)
 
@@ -149,16 +143,6 @@ class Metadata:  # pylint: disable=too-many-instance-attributes
         """Set message_type."""
         self.__dict__["_message_type"] = value
 
-    @property
-    def partition_id(self) -> int | None:
-        """An identifier telling which data partition a ClientApp should use."""
-        return cast(int, self.__dict__["_partition_id"])
-
-    @partition_id.setter
-    def partition_id(self, value: int) -> None:
-        """Set partition_id."""
-        self.__dict__["_partition_id"] = value
-
     def __repr__(self) -> str:
         """Return a string representation of this instance."""
         view = ", ".join([f"{k.lstrip('_')}={v!r}" for k, v in self.__dict__.items()])
@@ -398,5 +382,4 @@ def _create_reply_metadata(msg: Message, ttl: float) -> Metadata:
         group_id=msg.metadata.group_id,
         ttl=ttl,
         message_type=msg.metadata.message_type,
-        partition_id=msg.metadata.partition_id,
     )

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/src/py/flwr/server/superlink/fleet/vce/backend/raybackend.py

@@ -168,7 +168,7 @@ class RayBackend(Backend):
 
         Return output message and updated context.
         """
-        partition_id = message.metadata.partition_id
+        partition_id = context.partition_id
 
         try:
             # Submit a task to the pool

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/src/py/flwr/server/superlink/fleet/vce/vce_api.py

@@ -57,7 +57,6 @@ async def worker(
     queue: "asyncio.Queue[TaskIns]",
     node_states: Dict[int, NodeState],
     state_factory: StateFactory,
-    nodes_mapping: NodeToPartitionMapping,
     backend: Backend,
 ) -> None:
     """Get TaskIns from queue and pass it to an actor in the pool to execute it."""
@@ -74,8 +73,6 @@ async def worker(
 
             # Convert TaskIns to Message
             message = message_from_taskins(task_ins)
-            # Set partition_id
-            message.metadata.partition_id = nodes_mapping[node_id]
 
             # Let backend process message
             out_mssg, updated_context = await backend.process_message(
@@ -187,9 +184,7 @@ async def run(
         # Add workers (they submit Messages to Backend)
         worker_tasks = [
             asyncio.create_task(
-                worker(
-                    app_fn, queue, node_states, state_factory, nodes_mapping, backend
-                )
+                worker(app_fn, queue, node_states, state_factory, backend)
             )
             for _ in range(backend.num_workers)
         ]
@@ -291,8 +286,8 @@ def start_vce(
 
     # Construct mapping of NodeStates
     node_states: Dict[int, NodeState] = {}
-    for node_id in nodes_mapping:
-        node_states[node_id] = NodeState()
+    for node_id, partition_id in nodes_mapping.items():
+        node_states[node_id] = NodeState(partition_id=partition_id)
 
     # Load backend config
     log(DEBUG, "Supported backends: %s", list(supported_backends.keys()))

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/src/py/flwr/server/superlink/state/in_memory_state.py

@@ -15,7 +15,6 @@
 """In-memory State implementation."""
 
 
-import os
 import threading
 import time
 from logging import ERROR
@@ -23,12 +22,13 @@ from typing import Dict, List, Optional, Set, Tuple
 from uuid import UUID, uuid4
 
 from flwr.common import log, now
+from flwr.common.constant import NODE_ID_NUM_BYTES, RUN_ID_NUM_BYTES
 from flwr.common.typing import Run
 from flwr.proto.task_pb2 import TaskIns, TaskRes  # pylint: disable=E0611
 from flwr.server.superlink.state.state import State
 from flwr.server.utils import validate_task_ins_or_res
 
-from .utils import make_node_unavailable_taskres
+from .utils import generate_rand_int_from_bytes, make_node_unavailable_taskres
 
 
 class InMemoryState(State):  # pylint: disable=R0902,R0904
@@ -216,7 +216,7 @@ class InMemoryState(State):  # pylint: disable=R0902,R0904
     ) -> int:
         """Create, store in state, and return `node_id`."""
         # Sample a random int64 as node_id
-        node_id: int = int.from_bytes(os.urandom(8), "little", signed=True)
+        node_id = generate_rand_int_from_bytes(NODE_ID_NUM_BYTES)
 
         with self.lock:
             if node_id in self.node_ids:
@@ -279,7 +279,7 @@ class InMemoryState(State):  # pylint: disable=R0902,R0904
         """Create a new run for the specified `fab_id` and `fab_version`."""
         # Sample a random int64 as run_id
         with self.lock:
-            run_id: int = int.from_bytes(os.urandom(8), "little", signed=True)
+            run_id = generate_rand_int_from_bytes(RUN_ID_NUM_BYTES)
 
             if run_id not in self.run_ids:
                 self.run_ids[run_id] = Run(

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/src/py/flwr/server/superlink/state/sqlite_state.py

@@ -15,7 +15,6 @@
 """SQLite based implemenation of server state."""
 
 
-import os
 import re
 import sqlite3
 import time
@@ -24,6 +23,7 @@ from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union, cast
 from uuid import UUID, uuid4
 
 from flwr.common import log, now
+from flwr.common.constant import NODE_ID_NUM_BYTES, RUN_ID_NUM_BYTES
 from flwr.common.typing import Run
 from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
 from flwr.proto.recordset_pb2 import RecordSet  # pylint: disable=E0611
@@ -31,7 +31,7 @@ from flwr.proto.task_pb2 import Task, TaskIns, TaskRes  # pylint: disable=E0611
 from flwr.server.utils.validator import validate_task_ins_or_res
 
 from .state import State
-from .utils import make_node_unavailable_taskres
+from .utils import generate_rand_int_from_bytes, make_node_unavailable_taskres
 
 SQL_CREATE_TABLE_NODE = """
 CREATE TABLE IF NOT EXISTS node(
@@ -541,7 +541,7 @@ class SqliteState(State):  # pylint: disable=R0904
     ) -> int:
         """Create, store in state, and return `node_id`."""
         # Sample a random int64 as node_id
-        node_id: int = int.from_bytes(os.urandom(8), "little", signed=True)
+        node_id = generate_rand_int_from_bytes(NODE_ID_NUM_BYTES)
 
         query = "SELECT node_id FROM node WHERE public_key = :public_key;"
         row = self.query(query, {"public_key": public_key})
@@ -616,7 +616,7 @@ class SqliteState(State):  # pylint: disable=R0904
     def create_run(self, fab_id: str, fab_version: str) -> int:
         """Create a new run for the specified `fab_id` and `fab_version`."""
         # Sample a random int64 as run_id
-        run_id: int = int.from_bytes(os.urandom(8), "little", signed=True)
+        run_id = generate_rand_int_from_bytes(RUN_ID_NUM_BYTES)
 
         # Check conflicts
         query = "SELECT COUNT(*) FROM run WHERE run_id = ?;"

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/src/py/flwr/server/superlink/state/utils.py

@@ -17,6 +17,7 @@
 
 import time
 from logging import ERROR
+from os import urandom
 from uuid import uuid4
 
 from flwr.common import log
@@ -31,6 +32,11 @@ NODE_UNAVAILABLE_ERROR_REASON = (
 )
 
 
+def generate_rand_int_from_bytes(num_bytes: int) -> int:
+    """Generate a random `num_bytes` integer."""
+    return int.from_bytes(urandom(num_bytes), "little", signed=True)
+
+
 def make_node_unavailable_taskres(ref_taskins: TaskIns) -> TaskRes:
     """Generate a TaskRes with a node unavailable error from a TaskIns."""
     current_time = time.time()

{flwr_nightly-1.10.0.dev20240624 → flwr_nightly-1.10.0.dev20240629}/src/py/flwr/simulation/ray_transport/ray_client_proxy.py

@@ -53,7 +53,7 @@ class RayActorClientProxy(ClientProxy):
 
         self.app_fn = _load_app
         self.actor_pool = actor_pool
-        self.proxy_state = NodeState()
+        self.proxy_state = NodeState(partition_id=int(self.cid))
 
     def _submit_job(self, message: Message, timeout: Optional[float]) -> Message:
         """Sumbit a message to the ActorPool."""
@@ -107,7 +107,6 @@ class RayActorClientProxy(ClientProxy):
                 reply_to_message="",
                 ttl=timeout if timeout else DEFAULT_TTL,
                 message_type=message_type,
-                partition_id=int(self.cid),
             ),
         )
 

flwr_nightly-1.10.0.dev20240629/src/py/flwr/superexec/deployment.py

@@ -0,0 +1,109 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Deployment engine executor."""
+
+import subprocess
+import sys
+from logging import ERROR, INFO
+from typing import Optional
+
+from typing_extensions import override
+
+from flwr.cli.config_utils import get_fab_metadata
+from flwr.cli.install import install_from_fab
+from flwr.common.grpc import create_channel
+from flwr.common.logger import log
+from flwr.proto.driver_pb2 import CreateRunRequest  # pylint: disable=E0611
+from flwr.proto.driver_pb2_grpc import DriverStub
+from flwr.server.driver.grpc_driver import DEFAULT_SERVER_ADDRESS_DRIVER
+
+from .executor import Executor, RunTracker
+
+
+class DeploymentEngine(Executor):
+    """Deployment engine executor."""
+
+    def __init__(
+        self,
+        address: str = DEFAULT_SERVER_ADDRESS_DRIVER,
+        root_certificates: Optional[bytes] = None,
+    ) -> None:
+        self.address = address
+        self.root_certificates = root_certificates
+        self.stub: Optional[DriverStub] = None
+
+    def _connect(self) -> None:
+        if self.stub is None:
+            channel = create_channel(
+                server_address=self.address,
+                insecure=(self.root_certificates is None),
+                root_certificates=self.root_certificates,
+            )
+            self.stub = DriverStub(channel)
+
+    def _create_run(self, fab_id: str, fab_version: str) -> int:
+        if self.stub is None:
+            self._connect()
+
+        assert self.stub is not None
+
+        req = CreateRunRequest(fab_id=fab_id, fab_version=fab_version)
+        res = self.stub.CreateRun(request=req)
+        return int(res.run_id)
+
+    @override
+    def start_run(self, fab_file: bytes) -> Optional[RunTracker]:
+        """Start run using the Flower Deployment Engine."""
+        try:
+            # Install FAB to flwr dir
+            fab_version, fab_id = get_fab_metadata(fab_file)
+            fab_path = install_from_fab(fab_file, None, True)
+
+            # Install FAB Python package
+            subprocess.check_call(
+                [sys.executable, "-m", "pip", "install", str(fab_path)],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+
+            # Call SuperLink to create run
+            run_id: int = self._create_run(fab_id, fab_version)
+            log(INFO, "Created run %s", str(run_id))
+
+            # Start ServerApp
+            proc = subprocess.Popen(  # pylint: disable=consider-using-with
+                [
+                    "flower-server-app",
+                    "--run-id",
+                    str(run_id),
+                    "--insecure",
+                ],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+            )
+            log(INFO, "Started run %s", str(run_id))
+
+            return RunTracker(
+                run_id=run_id,
+                proc=proc,
+            )
+        # pylint: disable-next=broad-except
+        except Exception as e:
+            log(ERROR, "Could not start run: %s", str(e))
+            return None
+
+
+executor = DeploymentEngine()