syft-flwr 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

README.md

@@ -3,4 +3,6 @@
 `syft_flwr` is an open source framework that facilitate federated learning projects using [Flower](https://github.com/adap/flower) over the [SyftBox](https://github.com/OpenMined/syftbox) protocol
 
 ## Installation
-`pip install syft_flwr`
+- Install uv: `brew install uv`
+- Create a virtual environment: `uv venv`
+- Install `syft-flwr`: `uv pip install syft-flwr`

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "syft-flwr"
-version = "0.1.0"
-description = "Add your description here"
+version = "0.1.1"
+description = "syft_flwr is an open source framework that facilitate federated learning projects using Flower over the SyftBox protocol"
 readme = "README.md"
 requires-python = ">=3.9.2"
 dependencies = [
@@ -21,10 +21,6 @@ dependencies = [
 [project.scripts]
 syft_flwr = "syft_flwr.cli:main"
 
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
 [tool.uv]
 dev-dependencies = [
     "ipykernel>=6.29.5",
@@ -33,6 +29,10 @@ dev-dependencies = [
     "pre-commit>=4.0.1",
 ]
 
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
 [tool.hatch.build.targets.wheel]
 packages = ["src/syft_flwr"]
 only-include = ["src", "pyproject.toml", "/README.md"]

syft_flwr/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.0"
+__version__ = "0.1.1"
 
 from syft_flwr.bootstrap import bootstrap
 from syft_flwr.run import run

syft_flwr/bootstrap.py

@@ -1,3 +1,4 @@
+import time
 from pathlib import Path
 
 from loguru import logger
@@ -50,8 +51,15 @@ def __update_pyproject_toml(
     deps.append(f"syft_flwr=={__version__}")
     pyproject_conf["project"]["dependencies"] = deps
 
-    # always override the datasites and aggregator
     pyproject_conf["tool"]["syft_flwr"] = {}
+
+    # configure unique app name for each syft_flwr run
+    base_app_name = pyproject_conf["project"]["name"]
+    pyproject_conf["tool"]["syft_flwr"]["app_name"] = (
+        f"{aggregator}_{base_app_name}_{int(time.time())}"
+    )
+
+    # always override the datasites and aggregator
     pyproject_conf["tool"]["syft_flwr"]["datasites"] = datasites
     pyproject_conf["tool"]["syft_flwr"]["aggregator"] = aggregator
 

syft_flwr/flower_client.py

@@ -1,24 +1,53 @@
 import sys
 import traceback
 
-from flwr.client import ClientApp
-from flwr.common import Context
-from flwr.common.constant import ErrorCode
-from flwr.common.message import Error, Message
 from loguru import logger
 from syft_event import SyftEvents
 from syft_event.types import Request
 
+from flwr.client import ClientApp
+from flwr.common import Context
+from flwr.common.constant import ErrorCode, MessageType
+from flwr.common.message import Error, Message
 from syft_flwr.flwr_compatibility import RecordDict, create_flwr_message
 from syft_flwr.serde import bytes_to_flower_message, flower_message_to_bytes
 
 
-def syftbox_flwr_client(client_app: ClientApp, context: Context):
-    """Run the Flower ClientApp with SyftBox."""
+def _handle_normal_message(
+    message: Message, client_app: ClientApp, context: Context
+) -> bytes:
+    # Normal message handling
+    logger.info(f"Receive message with metadata: {message.metadata}")
+    reply_message: Message = client_app(message=message, context=context)
+    res_bytes: bytes = flower_message_to_bytes(reply_message)
+    logger.info(f"Reply message size: {len(res_bytes)/2**20} MB")
+    return res_bytes
+
 
-    box = SyftEvents("flwr")
+def _create_error_reply(message: Message, error: Error) -> bytes:
+    """Create and return error reply message in bytes."""
+    error_reply: Message = create_flwr_message(
+        content=RecordDict(),
+        reply_to=message,
+        message_type=message.metadata.message_type,
+        src_node_id=message.metadata.dst_node_id,
+        dst_node_id=message.metadata.src_node_id,
+        group_id=message.metadata.group_id,
+        run_id=message.metadata.run_id,
+        error=error,
+    )
+    error_bytes: bytes = flower_message_to_bytes(error_reply)
+    logger.info(f"Error reply message size: {len(error_bytes)/2**20} MB")
+    return error_bytes
+
+
+def syftbox_flwr_client(client_app: ClientApp, context: Context, app_name: str):
+    """Run the Flower ClientApp with SyftBox."""
+    syft_flwr_app_name = f"flwr/{app_name}"
+    box = SyftEvents(syft_flwr_app_name)
     client_email = box.client.email
     logger.info(f"Started SyftBox Flower Client on: {client_email}")
+    logger.info(f"syft_flwr app name: {syft_flwr_app_name}")
 
     @box.on_request("/messages")
     def handle_messages(request: Request) -> None:
@@ -26,12 +55,17 @@ def syftbox_flwr_client(client_app: ClientApp, context: Context):
             f"Received request id: {request.id}, size: {len(request.body) / 1024 / 1024} (MB)"
         )
         message: Message = bytes_to_flower_message(request.body)
-
         try:
-            reply_message: Message = client_app(message=message, context=context)
-            res_bytes: bytes = flower_message_to_bytes(reply_message)
-            logger.info(f"Reply message size: {len(res_bytes)/2**20} MB")
-            return res_bytes
+            # Handle stop signal
+            if (
+                message.metadata.message_type == MessageType.SYSTEM
+                and message.content["config"]["action"] == "stop"
+            ):
+                logger.info(f"Received stop message: {message}")
+                box._stop_event.set()
+                return None
+
+            return _handle_normal_message(message, client_app, context)
 
         except Exception as e:
             error_traceback = traceback.format_exc()
@@ -39,22 +73,10 @@ def syftbox_flwr_client(client_app: ClientApp, context: Context):
             logger.error(error_message)
 
             error = Error(
-                code=ErrorCode.CLIENT_APP_RAISED_EXCEPTION, reason=f"{error_message}"
-            )
-
-            error_reply: Message = create_flwr_message(
-                content=RecordDict(),
-                reply_to=message,
-                message_type=message.metadata.message_type,
-                src_node_id=message.metadata.dst_node_id,
-                dst_node_id=message.metadata.src_node_id,
-                group_id=message.metadata.group_id,
-                run_id=message.metadata.run_id,
-                error=error,
+                code=ErrorCode.CLIENT_APP_RAISED_EXCEPTION, reason=error_message
             )
-            error_bytes: bytes = flower_message_to_bytes(error_reply)
-            logger.info(f"Error reply message size: {len(error_bytes)/2**20} MB")
-            return error_bytes
+            box._stop_event.set()
+            return _create_error_reply(message, error)
 
     try:
         box.run_forever()

syft_flwr/flower_server.py

@@ -1,25 +1,42 @@
+import traceback
 from random import randint
 
+from loguru import logger
+
 from flwr.common import Context
 from flwr.server import ServerApp
 from flwr.server.run_serverapp import run as run_server
-from loguru import logger
-
 from syft_flwr.grid import SyftGrid
 
 
-def syftbox_flwr_server(server_app: ServerApp, context: Context, datasites: list[str]):
+def syftbox_flwr_server(
+    server_app: ServerApp,
+    context: Context,
+    datasites: list[str],
+    app_name: str,
+) -> Context:
     """Run the Flower ServerApp with SyftBox."""
-    syft_grid = SyftGrid(datasites=datasites)
+    syft_flwr_app_name = f"flwr/{app_name}"
+    syft_grid = SyftGrid(app_name=syft_flwr_app_name, datasites=datasites)
     run_id = randint(0, 1000)
     syft_grid.set_run(run_id)
     logger.info(f"Started SyftBox Flower Server on: {syft_grid._client.email}")
+    logger.info(f"syft_flwr app name: {syft_flwr_app_name}")
+
+    try:
+        updated_context = run_server(
+            syft_grid,
+            context=context,
+            loaded_server_app=server_app,
+            server_app_dir="",
+        )
+        logger.info(f"Server completed with context: {updated_context}")
+    except Exception as e:
+        logger.error(f"Server encountered an error: {str(e)}")
+        logger.error(f"Traceback: {traceback.format_exc()}")
+        updated_context = context
+    finally:
+        syft_grid.send_stop_signal(group_id="final", reason="Server stopped")
+        logger.info("Sending stop signals to the clients")
 
-    updated_context = run_server(
-        syft_grid,
-        context=context,
-        loaded_server_app=server_app,
-        server_app_dir="",
-    )
-    logger.info(f"Server completed with context: {updated_context}")
     return updated_context

syft_flwr/grid.py

@@ -1,14 +1,16 @@
 import time
 from typing import Iterable, cast
 
-from flwr.common.message import Message
-from flwr.common.typing import Run
-from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
 from loguru import logger
 from syft_core import Client
 from syft_rpc import rpc, rpc_db
 from typing_extensions import Optional
 
+from flwr.common import ConfigRecord
+from flwr.common.constant import MessageType
+from flwr.common.message import Message
+from flwr.common.typing import Run
+from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
 from syft_flwr.flwr_compatibility import (
     Grid,
     RecordDict,
@@ -25,6 +27,7 @@ AGGREGATOR_NODE_ID = 1
 class SyftGrid(Grid):
     def __init__(
         self,
+        app_name: str,
         datasites: list[str] = [],
         client: Client = None,
     ) -> None:
@@ -36,6 +39,7 @@ class SyftGrid(Grid):
         logger.debug(
             f"Initialize SyftGrid for '{self._client.email}' with datasites: {self.datasites}"
         )
+        self.app_name = app_name
 
     def set_run(self, run_id: int) -> None:
         # TODO: In Grpc Grid case, the superlink is the one which sets up the run id,
@@ -96,7 +100,9 @@ class SyftGrid(Grid):
             msg.metadata.__dict__["_src_node_id"] = self.node.node_id
             # RPC URL
             dest_datasite = self.client_map[msg.metadata.dst_node_id]
-            url = rpc.make_url(dest_datasite, app_name="flwr", endpoint="messages")
+            url = rpc.make_url(
+                dest_datasite, app_name=self.app_name, endpoint="messages"
+            )
             # Check message
             self._check_message(msg)
             # Serialize message
@@ -107,7 +113,9 @@ class SyftGrid(Grid):
                 f"Pushed message to {url} with metadata {msg.metadata}; size {len(msg_bytes) / 1024 / 1024} (Mb)"
             )
             # Save future
-            rpc_db.save_future(future=future, namespace="flwr", client=self._client)
+            rpc_db.save_future(
+                future=future, namespace=self.app_name, client=self._client
+            )
             message_ids.append(future.id)
 
         return message_ids
@@ -147,13 +155,13 @@ class SyftGrid(Grid):
         self,
         messages: Iterable[Message],
         *,
-        timeout: Optional[float] = None,
+        timeout: Optional[float] = 60,
     ) -> Iterable[Message]:
         """Push messages to specified node IDs and pull the reply messages.
 
         This method sends a list of messages to their destination node IDs and then
         waits for the replies. It continues to pull replies until either all replies are
-        received or the specified timeout duration is exceeded.
+        received or the specified timeout duration (in seconds) is exceeded.
         """
         # Push messages
         msg_ids = set(self.push_messages(messages))
@@ -165,7 +173,33 @@ class SyftGrid(Grid):
             res_msgs = self.pull_messages(msg_ids)
             ret.update(res_msgs)
             msg_ids.difference_update(res_msgs.keys())
-            if len(msg_ids) == 0:
+            if len(msg_ids) == 0:  # All messages received
                 break
-            time.sleep(3)
+            time.sleep(3)  # polling interval
+
+        if msg_ids:
+            logger.warning(
+                f"Timeout reached. {len(msg_ids)} message(s) sent out but not replied."
+            )
+
         return ret.values()
+
+    def send_stop_signal(
+        self, group_id: str, reason: str = "Training complete", ttl: float = 60.0
+    ) -> list[Message]:
+        """Send a stop signal to all datasites (clients)."""
+        stop_messages: list[Message] = [
+            self.create_message(
+                content=RecordDict(
+                    {"config": ConfigRecord({"action": "stop", "reason": reason})}
+                ),
+                message_type=MessageType.SYSTEM,
+                dst_node_id=node_id,
+                group_id=group_id,
+                ttl=ttl,
+            )
+            for node_id in self.get_node_ids()
+        ]
+        self.push_messages(stop_messages)
+
+        return stop_messages

syft_flwr/run.py

@@ -6,7 +6,6 @@ from flwr.client.client_app import LoadClientAppError
 from flwr.common import Context
 from flwr.common.object_ref import load_app
 from flwr.server.server_app import LoadServerAppError
-
 from syft_flwr.config import load_flwr_pyproject
 from syft_flwr.flower_client import syftbox_flwr_client
 from syft_flwr.flower_server import syftbox_flwr_server
@@ -23,6 +22,7 @@ warnings.filterwarnings("ignore", category=DeprecationWarning, module="pydantic"
 def syftbox_run_flwr_client(flower_project_dir: Path) -> None:
     pyproject_conf = load_flwr_pyproject(flower_project_dir)
     client_ref = pyproject_conf["tool"]["flwr"]["app"]["components"]["clientapp"]
+    app_name = pyproject_conf["tool"]["syft_flwr"]["app_name"]
 
     context = Context(
         run_id=uuid4().int,
@@ -37,13 +37,14 @@ def syftbox_run_flwr_client(flower_project_dir: Path) -> None:
         flower_project_dir,
     )
 
-    syftbox_flwr_client(client_app, context)
+    syftbox_flwr_client(client_app, context, app_name)
 
 
 def syftbox_run_flwr_server(flower_project_dir: Path) -> None:
     pyproject_conf = load_flwr_pyproject(flower_project_dir)
     datasites = pyproject_conf["tool"]["syft_flwr"]["datasites"]
     server_ref = pyproject_conf["tool"]["flwr"]["app"]["components"]["serverapp"]
+    app_name = pyproject_conf["tool"]["syft_flwr"]["app_name"]
 
     context = Context(
         run_id=uuid4().int,
@@ -58,4 +59,4 @@ def syftbox_run_flwr_server(flower_project_dir: Path) -> None:
         flower_project_dir,
     )
 
-    syftbox_flwr_server(server_app, context, datasites)
+    syftbox_flwr_server(server_app, context, datasites, app_name)

syft_flwr/strategy/fedavg.py

@@ -1,10 +1,11 @@
 from pathlib import Path
 
-from flwr.common import parameters_to_ndarrays
-from flwr.server.strategy import FedAvg
 from loguru import logger
 from safetensors.numpy import save_file
 
+from flwr.common import parameters_to_ndarrays
+from flwr.server.strategy import FedAvg
+
 
 class FedAvgWithModelSaving(FedAvg):
     """This is a custom strategy that behaves exactly like
@@ -23,9 +24,13 @@ class FedAvgWithModelSaving(FedAvg):
         ndarrays = parameters_to_ndarrays(parameters)
         tensor_dict = {f"layer_{i}": array for i, array in enumerate(ndarrays)}
         filename = self.save_path / f"parameters_round_{server_round}.safetensors"
-        save_file(tensor_dict, str(filename))
-
-        logger.info(f"Checkpoint saved to: {filename}")
+        if not self.save_path.exists():
+            logger.error(
+                f"Save directory {self.save_path} does NOT exist! Maybe it's deleted or moved."
+            )
+        else:
+            save_file(tensor_dict, str(filename))
+            logger.info(f"Checkpoint saved to: {filename}")
 
     def evaluate(self, server_round: int, parameters):
         """Evaluate model parameters using an evaluation function."""

{syft_flwr-0.1.0.dist-info → syft_flwr-0.1.1.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: syft-flwr
-Version: 0.1.0
-Summary: Add your description here
+Version: 0.1.1
+Summary: syft_flwr is an open source framework that facilitate federated learning projects using Flower over the SyftBox protocol
 License-File: LICENSE
 Requires-Python: >=3.9.2
 Requires-Dist: flwr-datasets[vision]>=0.5.0
@@ -22,4 +22,6 @@ Description-Content-Type: text/markdown
 `syft_flwr` is an open source framework that facilitate federated learning projects using [Flower](https://github.com/adap/flower) over the [SyftBox](https://github.com/OpenMined/syftbox) protocol
 
 ## Installation
-`pip install syft_flwr`
+- Install uv: `brew install uv`
+- Create a virtual environment: `uv venv`
+- Install `syft-flwr`: `uv pip install syft-flwr`

syft_flwr-0.1.1.dist-info/RECORD

@@ -0,0 +1,22 @@
+README.md,sha256=G_f6v-mRbjX7u1EU1IT8jDBk4QG7Z3FlBDajen3kB0Y,350
+pyproject.toml,sha256=WAErzoRIeSRx9uFGITECUwyyX1zNXe-x4XX5krrTr0U,1243
+syft_flwr/__init__.py,sha256=AgdUN00BHOIuh72ue7lHOjzcjHlYcGGlEjMUNhxJpIY,127
+syft_flwr/bootstrap.py,sha256=-T6SRh_p6u6uWpbTPZ6-URsAfMQAI2jakpjZAh0UUlw,3690
+syft_flwr/cli.py,sha256=imctwdQMxQeGQZaiKSX1Mo2nU_-RmA-cGB3H4huuUeA,3274
+syft_flwr/config.py,sha256=4hwkovGtFOLNULjJwoGYcA0uT4y3vZSrxndXqYXquMY,821
+syft_flwr/flower_client.py,sha256=jn0oTKf7ChSFd7DP5ZWi0P1BrNX6w2_Gcm4GqTxjIVM,3241
+syft_flwr/flower_server.py,sha256=sJwSEqePmkmWKGFXm2E44Ugoc6aaz-6tM7UaeWM2-co,1353
+syft_flwr/flwr_compatibility.py,sha256=vURf9rfsZ1uPm04szw6RpGYxtlG3BE4tW3YijptiGyk,3197
+syft_flwr/grid.py,sha256=Mh7OmXbHktcTBOlcKE3Ff97um6CxcwOdFp8G7C1htns,7202
+syft_flwr/run.py,sha256=OPW9bVt366DT-U-SxMpMLNXASwTZjp7XNNXfDP767f4,2153
+syft_flwr/run_simulation.py,sha256=t3shhfzAWDUlf6iQmwf5sS9APZQE9mkaZ9MLCYs9Ng0,6922
+syft_flwr/serde.py,sha256=5fCI-cRUOh5wE7cXQd4J6jex1grRGnyD1Jx-VlEDOXM,495
+syft_flwr/utils.py,sha256=3dDYEB7btq9hxZ9UsfQWh3i44OerAhGXc5XaX5wO3-o,955
+syft_flwr/strategy/__init__.py,sha256=mpUmExjjFkqU8gg41XsOBKfO3aqCBe7XPJSU-_P7smU,97
+syft_flwr/strategy/fedavg.py,sha256=N8jULUkjvuaBIEVINowyQln8W8yFhkO-J8k0-iPcGMA,1562
+syft_flwr/templates/main.py.tpl,sha256=p0uK97jvLGk3LJdy1_HF1R5BQgIjaTGkYnr-csfh39M,791
+syft_flwr-0.1.1.dist-info/METADATA,sha256=eF-zvd2wqmCtqbC9_1ULKSF8tgXPfE5DCILpuAqXBrQ,994
+syft_flwr-0.1.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+syft_flwr-0.1.1.dist-info/entry_points.txt,sha256=o7oT0dCoHn-3WyIwdDw1lBh2q-GvhB_8s0hWeJU4myc,49
+syft_flwr-0.1.1.dist-info/licenses/LICENSE,sha256=0msOUar8uPZTqkAOTBp4rCzd7Jl9eRhfKiNufwrsg7k,11361
+syft_flwr-0.1.1.dist-info/RECORD,,

syft_flwr-0.1.0.dist-info/RECORD

@@ -1,22 +0,0 @@
-README.md,sha256=rxa8da_OJlI6RVDAXa3tfmZC_dGEN_VFPJXJNMfXl-Q,250
-pyproject.toml,sha256=k7335xXFv_VDmvxmIk_wy6LriaTc92j0th88WDjJ-F0,1148
-syft_flwr/__init__.py,sha256=Tm2mxtuCAXctyyi89c9psk0PuGdv1TOauCg7-gFPyRk,127
-syft_flwr/bootstrap.py,sha256=P6bDLSnCQqrbxQ6-QBc0sR75L5ods8oLKJqItZzNlao,3446
-syft_flwr/cli.py,sha256=imctwdQMxQeGQZaiKSX1Mo2nU_-RmA-cGB3H4huuUeA,3274
-syft_flwr/config.py,sha256=4hwkovGtFOLNULjJwoGYcA0uT4y3vZSrxndXqYXquMY,821
-syft_flwr/flower_client.py,sha256=up15MSEBExiZPgcXHo1EIflWaVN0nO6y08gXrzpumSM,2418
-syft_flwr/flower_server.py,sha256=dz7dvEgFm76s8G266RfN46TohnMPGAk8K6DPWb23LsE,784
-syft_flwr/flwr_compatibility.py,sha256=vURf9rfsZ1uPm04szw6RpGYxtlG3BE4tW3YijptiGyk,3197
-syft_flwr/grid.py,sha256=76Wc1O1W8iGMr9QS97o9wTaU_87UB4fqYA5HnlIHy_U,6087
-syft_flwr/run.py,sha256=gLm_zjzCuWVNKVK2ocJzx_izXDJAW411nl6IXDyrkFM,2008
-syft_flwr/run_simulation.py,sha256=t3shhfzAWDUlf6iQmwf5sS9APZQE9mkaZ9MLCYs9Ng0,6922
-syft_flwr/serde.py,sha256=5fCI-cRUOh5wE7cXQd4J6jex1grRGnyD1Jx-VlEDOXM,495
-syft_flwr/utils.py,sha256=3dDYEB7btq9hxZ9UsfQWh3i44OerAhGXc5XaX5wO3-o,955
-syft_flwr/strategy/__init__.py,sha256=mpUmExjjFkqU8gg41XsOBKfO3aqCBe7XPJSU-_P7smU,97
-syft_flwr/strategy/fedavg.py,sha256=UCgXL_0woRjs7iHaqN16UYNJ7J6ogY3sO6620g1W3sU,1364
-syft_flwr/templates/main.py.tpl,sha256=p0uK97jvLGk3LJdy1_HF1R5BQgIjaTGkYnr-csfh39M,791
-syft_flwr-0.1.0.dist-info/METADATA,sha256=wnKYSmLIbSoKAd7A1Wk7PatwD_g5TuByahxzJYJitcc,799
-syft_flwr-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-syft_flwr-0.1.0.dist-info/entry_points.txt,sha256=o7oT0dCoHn-3WyIwdDw1lBh2q-GvhB_8s0hWeJU4myc,49
-syft_flwr-0.1.0.dist-info/licenses/LICENSE,sha256=0msOUar8uPZTqkAOTBp4rCzd7Jl9eRhfKiNufwrsg7k,11361
-syft_flwr-0.1.0.dist-info/RECORD,,