syft-flwr 0.1.0__tar.gz → 0.1.2__tar.gz

{syft_flwr-0.1.0 → syft_flwr-0.1.2}/.gitignore

@@ -182,6 +182,7 @@ checkpoints/
 # Datasets
 **MedMNIST**
 **/datasets/
+**/dataset/
 
 # Ruff cache
 .ruff_cache/

{syft_flwr-0.1.0 → syft_flwr-0.1.2}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: syft-flwr
-Version: 0.1.0
-Summary: Add your description here
+Version: 0.1.2
+Summary: syft_flwr is an open source framework that facilitate federated learning projects using Flower over the SyftBox protocol
 License-File: LICENSE
 Requires-Python: >=3.9.2
 Requires-Dist: flwr-datasets[vision]>=0.5.0
@@ -22,4 +22,6 @@ Description-Content-Type: text/markdown
 `syft_flwr` is an open source framework that facilitate federated learning projects using [Flower](https://github.com/adap/flower) over the [SyftBox](https://github.com/OpenMined/syftbox) protocol
 
 ## Installation
-`pip install syft_flwr`
+- Install uv: `brew install uv`
+- Create a virtual environment: `uv venv`
+- Install `syft-flwr`: `uv pip install syft-flwr`

{syft_flwr-0.1.0 → syft_flwr-0.1.2}/README.md

@@ -3,4 +3,6 @@
 `syft_flwr` is an open source framework that facilitate federated learning projects using [Flower](https://github.com/adap/flower) over the [SyftBox](https://github.com/OpenMined/syftbox) protocol
 
 ## Installation
-`pip install syft_flwr`
+- Install uv: `brew install uv`
+- Create a virtual environment: `uv venv`
+- Install `syft-flwr`: `uv pip install syft-flwr`

{syft_flwr-0.1.0 → syft_flwr-0.1.2}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "syft-flwr"
-version = "0.1.0"
-description = "Add your description here"
+version = "0.1.2"
+description = "syft_flwr is an open source framework that facilitate federated learning projects using Flower over the SyftBox protocol"
 readme = "README.md"
 requires-python = ">=3.9.2"
 dependencies = [
@@ -21,10 +21,6 @@ dependencies = [
 [project.scripts]
 syft_flwr = "syft_flwr.cli:main"
 
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
 [tool.uv]
 dev-dependencies = [
     "ipykernel>=6.29.5",
@@ -33,6 +29,10 @@ dev-dependencies = [
     "pre-commit>=4.0.1",
 ]
 
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
 [tool.hatch.build.targets.wheel]
 packages = ["src/syft_flwr"]
 only-include = ["src", "pyproject.toml", "/README.md"]

{syft_flwr-0.1.0 → syft_flwr-0.1.2}/src/syft_flwr/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.0"
+__version__ = "0.1.2"
 
 from syft_flwr.bootstrap import bootstrap
 from syft_flwr.run import run

{syft_flwr-0.1.0 → syft_flwr-0.1.2}/src/syft_flwr/bootstrap.py

@@ -1,3 +1,4 @@
+import time
 from pathlib import Path
 
 from loguru import logger
@@ -50,8 +51,15 @@ def __update_pyproject_toml(
     deps.append(f"syft_flwr=={__version__}")
     pyproject_conf["project"]["dependencies"] = deps
 
-    # always override the datasites and aggregator
     pyproject_conf["tool"]["syft_flwr"] = {}
+
+    # configure unique app name for each syft_flwr run
+    base_app_name = pyproject_conf["project"]["name"]
+    pyproject_conf["tool"]["syft_flwr"]["app_name"] = (
+        f"{aggregator}_{base_app_name}_{int(time.time())}"
+    )
+
+    # always override the datasites and aggregator
     pyproject_conf["tool"]["syft_flwr"]["datasites"] = datasites
     pyproject_conf["tool"]["syft_flwr"]["aggregator"] = aggregator
 

syft_flwr-0.1.2/src/syft_flwr/flower_client.py

@@ -0,0 +1,87 @@
+import sys
+import traceback
+
+from loguru import logger
+from syft_event import SyftEvents
+from syft_event.types import Request
+
+from flwr.client import ClientApp
+from flwr.common import Context
+from flwr.common.constant import ErrorCode, MessageType
+from flwr.common.message import Error, Message
+from syft_flwr.flwr_compatibility import RecordDict, create_flwr_message
+from syft_flwr.serde import bytes_to_flower_message, flower_message_to_bytes
+
+
+def _handle_normal_message(
+    message: Message, client_app: ClientApp, context: Context
+) -> bytes:
+    # Normal message handling
+    logger.info(f"Receive message with metadata: {message.metadata}")
+    reply_message: Message = client_app(message=message, context=context)
+    res_bytes: bytes = flower_message_to_bytes(reply_message)
+    logger.info(f"Reply message size: {len(res_bytes)/2**20} MB")
+    return res_bytes
+
+
+def _create_error_reply(message: Message, error: Error) -> bytes:
+    """Create and return error reply message in bytes."""
+    error_reply: Message = create_flwr_message(
+        content=RecordDict(),
+        reply_to=message,
+        message_type=message.metadata.message_type,
+        src_node_id=message.metadata.dst_node_id,
+        dst_node_id=message.metadata.src_node_id,
+        group_id=message.metadata.group_id,
+        run_id=message.metadata.run_id,
+        error=error,
+    )
+    error_bytes: bytes = flower_message_to_bytes(error_reply)
+    logger.info(f"Error reply message size: {len(error_bytes)/2**20} MB")
+    return error_bytes
+
+
+def syftbox_flwr_client(client_app: ClientApp, context: Context, app_name: str):
+    """Run the Flower ClientApp with SyftBox."""
+    syft_flwr_app_name = f"flwr/{app_name}"
+    box = SyftEvents(syft_flwr_app_name)
+    client_email = box.client.email
+    logger.info(f"Started SyftBox Flower Client on: {client_email}")
+    logger.info(f"syft_flwr app name: {syft_flwr_app_name}")
+
+    @box.on_request("/messages")
+    def handle_messages(request: Request) -> None:
+        logger.info(
+            f"Received request id: {request.id}, size: {len(request.body) / 1024 / 1024} (MB)"
+        )
+        message: Message = bytes_to_flower_message(request.body)
+        try:
+            # Handle stop signal
+            if (
+                message.metadata.message_type == MessageType.SYSTEM
+                and message.content["config"]["action"] == "stop"
+            ):
+                logger.info(f"Received stop message: {message}")
+                box._stop_event.set()
+                return None
+
+            return _handle_normal_message(message, client_app, context)
+
+        except Exception as e:
+            error_traceback = traceback.format_exc()
+            error_message = f"Client: '{client_email}'. Error: {str(e)}. Traceback: {error_traceback}"
+            logger.error(error_message)
+
+            error = Error(
+                code=ErrorCode.CLIENT_APP_RAISED_EXCEPTION, reason=error_message
+            )
+            box._stop_event.set()
+            return _create_error_reply(message, error)
+
+    try:
+        box.run_forever()
+    except Exception as e:
+        logger.error(
+            f"Fatal error in syftbox_flwr_client: {str(e)}\n{traceback.format_exc()}"
+        )
+        sys.exit(1)

syft_flwr-0.1.2/src/syft_flwr/flower_server.py

@@ -0,0 +1,42 @@
+import traceback
+from random import randint
+
+from loguru import logger
+
+from flwr.common import Context
+from flwr.server import ServerApp
+from flwr.server.run_serverapp import run as run_server
+from syft_flwr.grid import SyftGrid
+
+
+def syftbox_flwr_server(
+    server_app: ServerApp,
+    context: Context,
+    datasites: list[str],
+    app_name: str,
+) -> Context:
+    """Run the Flower ServerApp with SyftBox."""
+    syft_flwr_app_name = f"flwr/{app_name}"
+    syft_grid = SyftGrid(app_name=syft_flwr_app_name, datasites=datasites)
+    run_id = randint(0, 1000)
+    syft_grid.set_run(run_id)
+    logger.info(f"Started SyftBox Flower Server on: {syft_grid._client.email}")
+    logger.info(f"syft_flwr app name: {syft_flwr_app_name}")
+
+    try:
+        updated_context = run_server(
+            syft_grid,
+            context=context,
+            loaded_server_app=server_app,
+            server_app_dir="",
+        )
+        logger.info(f"Server completed with context: {updated_context}")
+    except Exception as e:
+        logger.error(f"Server encountered an error: {str(e)}")
+        logger.error(f"Traceback: {traceback.format_exc()}")
+        updated_context = context
+    finally:
+        syft_grid.send_stop_signal(group_id="final", reason="Server stopped")
+        logger.info("Sending stop signals to the clients")
+
+    return updated_context

{syft_flwr-0.1.0 → syft_flwr-0.1.2}/src/syft_flwr/grid.py

@@ -1,6 +1,9 @@
+import os
 import time
 from typing import Iterable, cast
 
+from flwr.common import ConfigRecord
+from flwr.common.constant import MessageType
 from flwr.common.message import Message
 from flwr.common.typing import Run
 from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
@@ -22,9 +25,14 @@ from syft_flwr.utils import str_to_int
 AGGREGATOR_NODE_ID = 1
 
 
+# env vars
+SYFT_FLWR_MSG_TIMEOUT = "SYFT_FLWR_MSG_TIMEOUT"
+
+
 class SyftGrid(Grid):
     def __init__(
         self,
+        app_name: str,
         datasites: list[str] = [],
         client: Client = None,
     ) -> None:
@@ -36,6 +44,7 @@ class SyftGrid(Grid):
         logger.debug(
             f"Initialize SyftGrid for '{self._client.email}' with datasites: {self.datasites}"
         )
+        self.app_name = app_name
 
     def set_run(self, run_id: int) -> None:
         # TODO: In Grpc Grid case, the superlink is the one which sets up the run id,
@@ -96,7 +105,9 @@ class SyftGrid(Grid):
             msg.metadata.__dict__["_src_node_id"] = self.node.node_id
             # RPC URL
             dest_datasite = self.client_map[msg.metadata.dst_node_id]
-            url = rpc.make_url(dest_datasite, app_name="flwr", endpoint="messages")
+            url = rpc.make_url(
+                dest_datasite, app_name=self.app_name, endpoint="messages"
+            )
             # Check message
             self._check_message(msg)
             # Serialize message
@@ -107,7 +118,9 @@ class SyftGrid(Grid):
                 f"Pushed message to {url} with metadata {msg.metadata}; size {len(msg_bytes) / 1024 / 1024} (Mb)"
             )
             # Save future
-            rpc_db.save_future(future=future, namespace="flwr", client=self._client)
+            rpc_db.save_future(
+                future=future, namespace=self.app_name, client=self._client
+            )
             message_ids.append(future.id)
 
         return message_ids
@@ -153,8 +166,19 @@ class SyftGrid(Grid):
 
         This method sends a list of messages to their destination node IDs and then
         waits for the replies. It continues to pull replies until either all replies are
-        received or the specified timeout duration is exceeded.
+        received or the specified timeout duration (in seconds) is exceeded.
         """
+        if os.environ.get(SYFT_FLWR_MSG_TIMEOUT) is not None:
+            timeout = float(os.environ.get(SYFT_FLWR_MSG_TIMEOUT))
+        if timeout is not None:
+            logger.debug(
+                f"syft_flwr messages timeout = {timeout}: Will move on after {timeout} (s) if no reply is received"
+            )
+        else:
+            logger.debug(
+                "syft_flwr messages timeout = None: Will wait indefinitely for replies"
+            )
+
         # Push messages
         msg_ids = set(self.push_messages(messages))
 
@@ -165,7 +189,33 @@ class SyftGrid(Grid):
             res_msgs = self.pull_messages(msg_ids)
             ret.update(res_msgs)
             msg_ids.difference_update(res_msgs.keys())
-            if len(msg_ids) == 0:
+            if len(msg_ids) == 0:  # All messages received
                 break
-            time.sleep(3)
+            time.sleep(3)  # polling interval
+
+        if msg_ids:
+            logger.warning(
+                f"Timeout reached. {len(msg_ids)} message(s) sent out but not replied."
+            )
+
         return ret.values()
+
+    def send_stop_signal(
+        self, group_id: str, reason: str = "Training complete", ttl: float = 60.0
+    ) -> list[Message]:
+        """Send a stop signal to all datasites (clients)."""
+        stop_messages: list[Message] = [
+            self.create_message(
+                content=RecordDict(
+                    {"config": ConfigRecord({"action": "stop", "reason": reason})}
+                ),
+                message_type=MessageType.SYSTEM,
+                dst_node_id=node_id,
+                group_id=group_id,
+                ttl=ttl,
+            )
+            for node_id in self.get_node_ids()
+        ]
+        self.push_messages(stop_messages)
+
+        return stop_messages

{syft_flwr-0.1.0 → syft_flwr-0.1.2}/src/syft_flwr/run.py

@@ -6,7 +6,6 @@ from flwr.client.client_app import LoadClientAppError
 from flwr.common import Context
 from flwr.common.object_ref import load_app
 from flwr.server.server_app import LoadServerAppError
-
 from syft_flwr.config import load_flwr_pyproject
 from syft_flwr.flower_client import syftbox_flwr_client
 from syft_flwr.flower_server import syftbox_flwr_server
@@ -23,6 +22,7 @@ warnings.filterwarnings("ignore", category=DeprecationWarning, module="pydantic"
 def syftbox_run_flwr_client(flower_project_dir: Path) -> None:
     pyproject_conf = load_flwr_pyproject(flower_project_dir)
     client_ref = pyproject_conf["tool"]["flwr"]["app"]["components"]["clientapp"]
+    app_name = pyproject_conf["tool"]["syft_flwr"]["app_name"]
 
     context = Context(
         run_id=uuid4().int,
@@ -37,13 +37,14 @@ def syftbox_run_flwr_client(flower_project_dir: Path) -> None:
         flower_project_dir,
     )
 
-    syftbox_flwr_client(client_app, context)
+    syftbox_flwr_client(client_app, context, app_name)
 
 
 def syftbox_run_flwr_server(flower_project_dir: Path) -> None:
     pyproject_conf = load_flwr_pyproject(flower_project_dir)
     datasites = pyproject_conf["tool"]["syft_flwr"]["datasites"]
     server_ref = pyproject_conf["tool"]["flwr"]["app"]["components"]["serverapp"]
+    app_name = pyproject_conf["tool"]["syft_flwr"]["app_name"]
 
     context = Context(
         run_id=uuid4().int,
@@ -58,4 +59,4 @@ def syftbox_run_flwr_server(flower_project_dir: Path) -> None:
         flower_project_dir,
     )
 
-    syftbox_flwr_server(server_app, context, datasites)
+    syftbox_flwr_server(server_app, context, datasites, app_name)

{syft_flwr-0.1.0 → syft_flwr-0.1.2}/src/syft_flwr/strategy/fedavg.py

@@ -1,10 +1,11 @@
 from pathlib import Path
 
-from flwr.common import parameters_to_ndarrays
-from flwr.server.strategy import FedAvg
 from loguru import logger
 from safetensors.numpy import save_file
 
+from flwr.common import parameters_to_ndarrays
+from flwr.server.strategy import FedAvg
+
 
 class FedAvgWithModelSaving(FedAvg):
     """This is a custom strategy that behaves exactly like
@@ -23,9 +24,13 @@ class FedAvgWithModelSaving(FedAvg):
         ndarrays = parameters_to_ndarrays(parameters)
         tensor_dict = {f"layer_{i}": array for i, array in enumerate(ndarrays)}
         filename = self.save_path / f"parameters_round_{server_round}.safetensors"
-        save_file(tensor_dict, str(filename))
-
-        logger.info(f"Checkpoint saved to: {filename}")
+        if not self.save_path.exists():
+            logger.error(
+                f"Save directory {self.save_path} does NOT exist! Maybe it's deleted or moved."
+            )
+        else:
+            save_file(tensor_dict, str(filename))
+            logger.info(f"Checkpoint saved to: {filename}")
 
     def evaluate(self, server_round: int, parameters):
         """Evaluate model parameters using an evaluation function."""

syft_flwr-0.1.0/src/syft_flwr/flower_client.py

@@ -1,65 +0,0 @@
-import sys
-import traceback
-
-from flwr.client import ClientApp
-from flwr.common import Context
-from flwr.common.constant import ErrorCode
-from flwr.common.message import Error, Message
-from loguru import logger
-from syft_event import SyftEvents
-from syft_event.types import Request
-
-from syft_flwr.flwr_compatibility import RecordDict, create_flwr_message
-from syft_flwr.serde import bytes_to_flower_message, flower_message_to_bytes
-
-
-def syftbox_flwr_client(client_app: ClientApp, context: Context):
-    """Run the Flower ClientApp with SyftBox."""
-
-    box = SyftEvents("flwr")
-    client_email = box.client.email
-    logger.info(f"Started SyftBox Flower Client on: {client_email}")
-
-    @box.on_request("/messages")
-    def handle_messages(request: Request) -> None:
-        logger.info(
-            f"Received request id: {request.id}, size: {len(request.body) / 1024 / 1024} (MB)"
-        )
-        message: Message = bytes_to_flower_message(request.body)
-
-        try:
-            reply_message: Message = client_app(message=message, context=context)
-            res_bytes: bytes = flower_message_to_bytes(reply_message)
-            logger.info(f"Reply message size: {len(res_bytes)/2**20} MB")
-            return res_bytes
-
-        except Exception as e:
-            error_traceback = traceback.format_exc()
-            error_message = f"Client: '{client_email}'. Error: {str(e)}. Traceback: {error_traceback}"
-            logger.error(error_message)
-
-            error = Error(
-                code=ErrorCode.CLIENT_APP_RAISED_EXCEPTION, reason=f"{error_message}"
-            )
-
-            error_reply: Message = create_flwr_message(
-                content=RecordDict(),
-                reply_to=message,
-                message_type=message.metadata.message_type,
-                src_node_id=message.metadata.dst_node_id,
-                dst_node_id=message.metadata.src_node_id,
-                group_id=message.metadata.group_id,
-                run_id=message.metadata.run_id,
-                error=error,
-            )
-            error_bytes: bytes = flower_message_to_bytes(error_reply)
-            logger.info(f"Error reply message size: {len(error_bytes)/2**20} MB")
-            return error_bytes
-
-    try:
-        box.run_forever()
-    except Exception as e:
-        logger.error(
-            f"Fatal error in syftbox_flwr_client: {str(e)}\n{traceback.format_exc()}"
-        )
-        sys.exit(1)

syft_flwr-0.1.0/src/syft_flwr/flower_server.py

@@ -1,25 +0,0 @@
-from random import randint
-
-from flwr.common import Context
-from flwr.server import ServerApp
-from flwr.server.run_serverapp import run as run_server
-from loguru import logger
-
-from syft_flwr.grid import SyftGrid
-
-
-def syftbox_flwr_server(server_app: ServerApp, context: Context, datasites: list[str]):
-    """Run the Flower ServerApp with SyftBox."""
-    syft_grid = SyftGrid(datasites=datasites)
-    run_id = randint(0, 1000)
-    syft_grid.set_run(run_id)
-    logger.info(f"Started SyftBox Flower Server on: {syft_grid._client.email}")
-
-    updated_context = run_server(
-        syft_grid,
-        context=context,
-        loaded_server_app=server_app,
-        server_app_dir="",
-    )
-    logger.info(f"Server completed with context: {updated_context}")
-    return updated_context