syft-flwr 0.2.2__tar.gz → 0.4.1__tar.gz

{syft_flwr-0.2.2 → syft_flwr-0.4.1}/PKG-INFO

@@ -1,16 +1,16 @@
 Metadata-Version: 2.4
 Name: syft-flwr
-Version: 0.2.2
+Version: 0.4.1
 Summary: syft_flwr is an open source framework that facilitate federated learning projects using Flower over the SyftBox protocol
 License-File: LICENSE
-Requires-Python: >=3.10
+Requires-Python: <3.14,>=3.11
 Requires-Dist: flwr-datasets[vision]>=0.5.0
 Requires-Dist: flwr[simulation]==1.21.0
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: safetensors>=0.6.2
-Requires-Dist: syft-rds>=0.2.2
+Requires-Dist: syft-rds>=0.4.2
 Requires-Dist: tomli-w>=1.2.0
-Requires-Dist: tomli>=2.2.1
+Requires-Dist: tomli>=2.3.0
 Requires-Dist: typing-extensions>=4.13.0
 Description-Content-Type: text/markdown
 
@@ -18,10 +18,14 @@ Description-Content-Type: text/markdown
 
 `syft_flwr` is an open source framework that facilitate federated learning (FL) projects using [Flower](https://github.com/adap/flower) over the [SyftBox](https://github.com/OpenMined/syftbox) protocol
 
-![FL Training Process](notebooks/fl-diabetes-prediction/images/fltraining.gif)
+![FL Training Process](https://github.com/OpenMined/syft-flwr/raw/main/notebooks/fl-diabetes-prediction/images/fltraining.gif)
 
 ## Example Usages
 Please look at the `notebooks/` folder for example use cases:
 -  [FL diabetes prediction](notebooks/fl-diabetes-prediction/README.md) shows how to train a federated model over distributed machines for multiple rounds
 -  [Federated analytics](notebooks/federated-analytics-diabetes/README.md) shows how to query statistics from private datasets from distributed machines and then aggregate them
--  [FedRAG (Federated RAG)](notebooks/fedrag/README.md) demonstrates privacy-preserving question answering using Retrieval Augmented Generation across distributed document sources with remote data science workflow
+-  [FedRAG (Federated RAG)](notebooks/fedrag/README.md) demonstrates privacy-preserving question answering using Retrieval Augmented Generation across distributed document sources with remote data science workflow
+
+## Development
+### Releasing
+See [RELEASE.md](RELEASE.md) for the complete release process.

{syft_flwr-0.2.2 → syft_flwr-0.4.1}/README.md

@@ -2,10 +2,14 @@
 
 `syft_flwr` is an open source framework that facilitate federated learning (FL) projects using [Flower](https://github.com/adap/flower) over the [SyftBox](https://github.com/OpenMined/syftbox) protocol
 
-![FL Training Process](notebooks/fl-diabetes-prediction/images/fltraining.gif)
+![FL Training Process](https://github.com/OpenMined/syft-flwr/raw/main/notebooks/fl-diabetes-prediction/images/fltraining.gif)
 
 ## Example Usages
 Please look at the `notebooks/` folder for example use cases:
 -  [FL diabetes prediction](notebooks/fl-diabetes-prediction/README.md) shows how to train a federated model over distributed machines for multiple rounds
 -  [Federated analytics](notebooks/federated-analytics-diabetes/README.md) shows how to query statistics from private datasets from distributed machines and then aggregate them
--  [FedRAG (Federated RAG)](notebooks/fedrag/README.md) demonstrates privacy-preserving question answering using Retrieval Augmented Generation across distributed document sources with remote data science workflow
+-  [FedRAG (Federated RAG)](notebooks/fedrag/README.md) demonstrates privacy-preserving question answering using Retrieval Augmented Generation across distributed document sources with remote data science workflow
+
+## Development
+### Releasing
+See [RELEASE.md](RELEASE.md) for the complete release process.

{syft_flwr-0.2.2 → syft_flwr-0.4.1}/pyproject.toml

@@ -1,29 +1,29 @@
 [project]
 name = "syft-flwr"
-version = "0.2.2"
+version = "0.4.1"
 description = "syft_flwr is an open source framework that facilitate federated learning projects using Flower over the SyftBox protocol"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.11,<3.14"
 dependencies = [
-    "syft-rds>=0.2.2",
+    "syft-rds>=0.4.2",
     "flwr[simulation]==1.21.0",
     "flwr-datasets[vision]>=0.5.0",
     "loguru>=0.7.3",
     "safetensors>=0.6.2",
     "typing-extensions>=4.13.0",
-    "tomli>=2.2.1",
+    "tomli>=2.3.0",
     "tomli-w>=1.2.0",
 ]
 
 # [tool.uv.sources]
-# syft-rds = { path = "../syft-data-science" }  # for development
-
+# syft-rds = { git = "https://github.com/OpenMined/syft-rds/", branch = "feature/python-runtime-uv-envs" }  # for development
+# syft-rds = { path = "../syft-rds", editable = true }  # for development
 
 [project.scripts]
 syft_flwr = "syft_flwr.cli:main"
 
-[tool.uv]
-dev-dependencies = [
+[dependency-groups]
+dev = [
     "ipykernel>=6.30.1",
     "ipywidgets>=8.1.7",
     "pytest>=8.4.1",
@@ -33,6 +33,7 @@ dev-dependencies = [
     "jupyterlab",
     "pytest-xdist>=3.8.0",
     "pytest-asyncio>=1.1.0",
+    "commitizen>=4.9.1",
 ]
 
 [build-system]
@@ -56,3 +57,14 @@ extend-select = ["I"]
 
 [tool.ruff.lint.per-file-ignores]
 "**/__init__.py" = ["F401"]
+
+[tool.commitizen]
+name = "cz_conventional_commits"
+version_provider = "pep621"
+tag_format = "v$version"
+update_changelog_on_bump = false
+version_files = [
+    "pyproject.toml:^version\\s*=\\s*\"(?P<version>.*)\"$",
+    "src/syft_flwr/__init__.py:^__version__\\s*=\\s*\"(?P<version>.*)\"$",
+    "notebooks/*/pyproject.toml:\"syft-flwr>=(?P<version>[^\"]+)\"",
+]

{syft_flwr-0.2.2 → syft_flwr-0.4.1}/src/syft_flwr/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.2.2"
+__version__ = "0.4.1"
 
 from syft_flwr.bootstrap import bootstrap
 from syft_flwr.run import run

{syft_flwr-0.2.2 → syft_flwr-0.4.1}/src/syft_flwr/bootstrap.py

@@ -101,5 +101,5 @@ def bootstrap(
     __copy_main_py(flwr_project_dir)
 
     logger.info(
-        f"Successfully bootstrapped syft-flwr project at {flwr_project_dir} with datasites {datasites} and aggregator {aggregator} ✅"
+        f"Successfully bootstrapped syft-flwr project at {flwr_project_dir} with datasites {datasites} and aggregator '{aggregator}' ✅"
     )

{syft_flwr-0.2.2 → syft_flwr-0.4.1}/src/syft_flwr/flower_client.py

@@ -49,7 +49,9 @@ class MessageHandler:
         error_reply = create_flwr_message(
             content=RecordDict(),
             reply_to=message,
-            message_type=message.metadata.message_type if message else MessageType.TASK,
+            message_type=message.metadata.message_type
+            if message
+            else MessageType.SYSTEM,
             dst_node_id=message.metadata.src_node_id if message else 0,
             group_id=message.metadata.group_id if message else "",
             error=error,
@@ -127,11 +129,15 @@ class RequestProcessor:
             logger.error(
                 f"❌ Failed to deserialize message from {original_sender}: {e}"
             )
-            error = Error(
-                code=ErrorCode.CLIENT_APP_RAISED_EXCEPTION,
-                reason=f"Message deserialization failed: {e}",
+            logger.debug(
+                f"Request body preview (first 200 bytes): {str(request.body[:200])}"
+            )
+
+            # Can't create error reply without valid message - skip response
+            logger.warning(
+                "Skipping error reply (cannot create without valid parsed message)"
             )
-            return self.message_handler.create_error_reply(None, error)
+            return None
 
         # Handle message
         try:
@@ -151,7 +157,6 @@ class RequestProcessor:
             error = Error(
                 code=ErrorCode.CLIENT_APP_RAISED_EXCEPTION, reason=error_message
             )
-            self.box._stop_event.set()
             return self.message_handler.create_error_reply(message, error)
 
 
@@ -159,11 +164,20 @@ def syftbox_flwr_client(client_app: ClientApp, context: Context, app_name: str):
     """Run the Flower ClientApp with SyftBox."""
     # Setup
     client, encryption_enabled, syft_flwr_app_name = setup_client(app_name)
-    box = SyftEvents(app_name=syft_flwr_app_name, client=client)
+    box = SyftEvents(
+        app_name=syft_flwr_app_name,
+        client=client,
+        cleanup_expiry="1d",  # Keep request/response files for 1 days
+        cleanup_interval="1d",  # Run cleanup daily
+    )
 
     logger.info(f"Started SyftBox Flower Client on: {box.client.email}")
     logger.info(f"syft_flwr app name: {syft_flwr_app_name}")
 
+    # Check if cleanup is running
+    if box.is_cleanup_running():
+        logger.info("Cleanup service is active")
+
     # Create handlers
     message_handler = MessageHandler(client_app, context, encryption_enabled)
     processor = RequestProcessor(message_handler, box, box.client.email)

{syft_flwr-0.2.2 → syft_flwr-0.4.1}/src/syft_flwr/grid.py

@@ -169,16 +169,19 @@ class SyftGrid(Grid):
 
         return message_ids
 
-    def pull_messages(self, message_ids: List[str]) -> Dict[str, Message]:
+    def pull_messages(self, message_ids: List[str]) -> Tuple[Dict[str, Message], set]:
         """Pull response messages from clients using future IDs.
 
         Args:
             message_ids: List of future IDs from push_messages()
 
         Returns:
-            Dict mapping message_id to Flower Message response
+            Tuple of:
+            - Dict mapping message_id to Flower Message response (includes both successes and client errors)
+            - Set of message_ids that are completed (got response, deserialized successfully, or permanently failed)
         """
         messages = {}
+        completed_ids = set()
 
         for msg_id in message_ids:
             try:
@@ -194,9 +197,15 @@ class SyftGrid(Grid):
                 # Process the response
                 message = self._process_response(response, msg_id)
 
+                # Always delete the future once we get a response (success or error)
+                # This prevents retrying failed messages indefinitely
+                rpc_db.delete_future(future_id=msg_id, client=self._client)
+
+                # Mark as completed regardless of success/failure
+                completed_ids.add(msg_id)
+
                 if message:
                     messages[msg_id] = message
-                    rpc_db.delete_future(future_id=msg_id, client=self._client)
 
             except Exception as e:
                 logger.error(f"❌ Unexpected error pulling message {msg_id}: {e}")
@@ -205,7 +214,7 @@ class SyftGrid(Grid):
         # Log summary
         self._log_pull_summary(messages, message_ids)
 
-        return messages
+        return messages, completed_ids
 
     def send_and_receive(
         self,
@@ -448,9 +457,10 @@ class SyftGrid(Grid):
 
         while pending_ids and (timeout is None or time.time() < end_time):
             # Pull available messages
-            batch = self.pull_messages(pending_ids)
+            batch, completed = self.pull_messages(list(pending_ids))
             responses.update(batch)
-            pending_ids.difference_update(batch.keys())
+            # Remove all completed IDs (both successes and failures)
+            pending_ids.difference_update(completed)
 
             if pending_ids:
                 time.sleep(poll_interval)  # Configurable polling interval
@@ -487,25 +497,25 @@ class SyftGrid(Grid):
             )
             return None
 
-        # Check for errors in message
+        # Check for errors in message (but still return it so Flower can handle the failure)
         if message.has_error():
             error = message.error
             logger.error(
                 f"❌ Message {msg_id} returned error with code={error.code}, "
-                f"reason={error.reason}"
+                f"reason={error.reason}. Returning error message to Flower for proper failure handling."
+            )
+        else:
+            # Log successful pull only if no error
+            encryption_status = (
+                "🔐 ENCRYPTED" if self._encryption_enabled else "📥 PLAINTEXT"
+            )
+            logger.debug(
+                f"{encryption_status} Pulled message from {response.url} "
+                f"with metadata: {message.metadata}, "
+                f"size: {len(response_body) / 1024 / 1024:.2f} MB"
             )
-            return None
-
-        # Log successful pull
-        encryption_status = (
-            "🔐 ENCRYPTED" if self._encryption_enabled else "📥 PLAINTEXT"
-        )
-        logger.debug(
-            f"{encryption_status} Pulled message from {response.url} "
-            f"with metadata: {message.metadata}, "
-            f"size: {len(response_body) / 1024 / 1024:.2f} MB"
-        )
 
+        # Always return the message (even with errors) so Flower's strategy can handle failures
         return message
 
     def _try_decrypt_response(self, body: bytes, msg_id: str) -> bytes:
@@ -545,14 +555,26 @@ class SyftGrid(Grid):
             )
 
     def _get_timeout(self, timeout: Optional[float]) -> Optional[float]:
-        """Get timeout value from environment or parameter."""
+        """Get timeout value from environment or parameter.
+
+        Priority:
+        1. Explicit timeout parameter
+        2. SYFT_FLWR_MSG_TIMEOUT environment variable
+        3. Default: 120 seconds (to prevent indefinite waiting)
+        """
+        # First check explicit parameter
+        if timeout is not None:
+            logger.debug(f"Message timeout: {timeout}s (from parameter)")
+            return timeout
+
+        # Then check environment variable
         env_timeout = os.environ.get(SYFT_FLWR_MSG_TIMEOUT)
         if env_timeout is not None:
             timeout = float(env_timeout)
+            logger.debug(f"Message timeout: {timeout}s (from env var)")
+            return timeout
 
-        if timeout is not None:
-            logger.debug(f"Message timeout: {timeout}s")
-        else:
-            logger.debug("No timeout - will wait indefinitely for replies")
-
-        return timeout
+        # Default to 120 seconds to prevent indefinite waiting
+        default_timeout = 120.0
+        logger.debug(f"Message timeout: {default_timeout}s (default)")
+        return default_timeout

{syft_flwr-0.2.2 → syft_flwr-0.4.1}/src/syft_flwr/run_simulation.py

@@ -1,20 +1,42 @@
 import asyncio
 import os
+import shutil
 import sys
 import tempfile
 from pathlib import Path
+from typing import TypeAlias
 
 from loguru import logger
 from syft_core import Client
 from syft_crypto import did_path, ensure_bootstrap, get_did_document, private_key_path
+from syft_rds import init_session
 from syft_rds.client.rds_client import RDSClient
-from syft_rds.orchestra import SingleRDSStack, remove_rds_stack_dir
 from typing_extensions import Optional, Union
 
 from syft_flwr.config import load_flwr_pyproject
 from syft_flwr.consts import SYFT_FLWR_ENCRYPTION_ENABLED
 from syft_flwr.utils import create_temp_client
 
+PathLike: TypeAlias = Union[str, os.PathLike, Path]
+
+
+def remove_rds_stack_dir(
+    key: str = "shared_client_dir", root_dir: Optional[PathLike] = None
+) -> None:
+    root_path = (
+        Path(root_dir).resolve() / key if root_dir else Path(tempfile.gettempdir(), key)
+    )
+
+    if not root_path.exists():
+        logger.warning(f"⚠️ Skipping removal, as path {root_path} does not exist")
+        return None
+
+    try:
+        shutil.rmtree(root_path)
+        logger.info(f"✅ Successfully removed directory {root_path}")
+    except Exception as e:
+        logger.error(f"❌ Failed to remove directory {root_path}: {e}")
+
 
 def _setup_mock_rds_clients(
     project_dir: Path, aggregator: str, datasites: list[str]
@@ -26,16 +48,18 @@ def _setup_mock_rds_clients(
     ds_syftbox_client = create_temp_client(
         email=aggregator, workspace_dir=simulated_syftbox_network_dir
     )
-    ds_stack = SingleRDSStack(client=ds_syftbox_client)
-    ds_rds_client = ds_stack.init_session(host=aggregator)
+    ds_rds_client = init_session(
+        host=aggregator, email=aggregator, syftbox_client=ds_syftbox_client
+    )
 
     do_rds_clients = []
     for datasite in datasites:
         do_syftbox_client = create_temp_client(
             email=datasite, workspace_dir=simulated_syftbox_network_dir
         )
-        do_stack = SingleRDSStack(client=do_syftbox_client)
-        do_rds_client = do_stack.init_session(host=datasite)
+        do_rds_client = init_session(
+            host=datasite, email=datasite, syftbox_client=do_syftbox_client
+        )
         do_rds_clients.append(do_rds_client)
 
     return simulated_syftbox_network_dir, do_rds_clients, ds_rds_client

{syft_flwr-0.2.2 → syft_flwr-0.4.1}/src/syft_flwr/templates/main.py.tpl

@@ -1,4 +1,5 @@
 import os
+import sys
 from pathlib import Path
 
 from syft_core import Client
@@ -20,8 +21,11 @@ is_server = client.email in config["tool"]["syft_flwr"]["aggregator"]
 if is_client:
     # run by each DO
     syftbox_run_flwr_client(flower_project_dir)
+    sys.exit(0)  # Exit cleanly after client work completes
 elif is_server:
     # run by the DS
     syftbox_run_flwr_server(flower_project_dir)
+    sys.exit(0)  # Exit cleanly after server work completes
 else:
     raise ValueError(f"{client.email} is not in config.datasites or config.aggregator")
+    sys.exit(1)  # Exit with error code

{syft_flwr-0.2.2 → syft_flwr-0.4.1}/src/syft_flwr/utils.py

@@ -104,8 +104,19 @@ def create_flwr_message(
             return Message(reply_to=reply_to, error=error)
         return Message(content=content, reply_to=reply_to)
     else:
+        # Allow standalone error messages when we can't parse the original message
         if error is not None:
-            raise ValueError("Error and reply_to cannot both be None")
+            logger.warning(
+                "Creating error message without reply_to (failed to parse request)"
+            )
+            return Message(
+                content=RecordDict(),
+                dst_node_id=dst_node_id,
+                message_type=message_type,
+                ttl=ttl,
+                group_id=group_id,
+                error=error,
+            )
         return Message(
             content=content,
             dst_node_id=dst_node_id,