syft-flwr 0.2.0__tar.gz → 0.2.2__tar.gz

{syft_flwr-0.2.0 → syft_flwr-0.2.2}/PKG-INFO

@@ -1,14 +1,14 @@
 Metadata-Version: 2.4
 Name: syft-flwr
-Version: 0.2.0
+Version: 0.2.2
 Summary: syft_flwr is an open source framework that facilitate federated learning projects using Flower over the SyftBox protocol
 License-File: LICENSE
 Requires-Python: >=3.10
 Requires-Dist: flwr-datasets[vision]>=0.5.0
-Requires-Dist: flwr[simulation]>=1.20.0
+Requires-Dist: flwr[simulation]==1.21.0
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: safetensors>=0.6.2
-Requires-Dist: syft-rds>=0.2.1
+Requires-Dist: syft-rds>=0.2.2
 Requires-Dist: tomli-w>=1.2.0
 Requires-Dist: tomli>=2.2.1
 Requires-Dist: typing-extensions>=4.13.0
@@ -23,4 +23,5 @@ Description-Content-Type: text/markdown
 ## Example Usages
 Please look at the `notebooks/` folder for example use cases:
 -  [FL diabetes prediction](notebooks/fl-diabetes-prediction/README.md) shows how to train a federated model over distributed machines for multiple rounds
--  [Federated analytics](notebooks/federated-analytics-diabetes/README.md) shows how to query statistics from private datasets from distributed machines and then aggregate them
+-  [Federated analytics](notebooks/federated-analytics-diabetes/README.md) shows how to query statistics from private datasets from distributed machines and then aggregate them
+-  [FedRAG (Federated RAG)](notebooks/fedrag/README.md) demonstrates privacy-preserving question answering using Retrieval Augmented Generation across distributed document sources with remote data science workflow

{syft_flwr-0.2.0 → syft_flwr-0.2.2}/README.md

@@ -7,4 +7,5 @@
 ## Example Usages
 Please look at the `notebooks/` folder for example use cases:
 -  [FL diabetes prediction](notebooks/fl-diabetes-prediction/README.md) shows how to train a federated model over distributed machines for multiple rounds
--  [Federated analytics](notebooks/federated-analytics-diabetes/README.md) shows how to query statistics from private datasets from distributed machines and then aggregate them
+-  [Federated analytics](notebooks/federated-analytics-diabetes/README.md) shows how to query statistics from private datasets from distributed machines and then aggregate them
+-  [FedRAG (Federated RAG)](notebooks/fedrag/README.md) demonstrates privacy-preserving question answering using Retrieval Augmented Generation across distributed document sources with remote data science workflow

{syft_flwr-0.2.0 → syft_flwr-0.2.2}/pyproject.toml

@@ -1,12 +1,12 @@
 [project]
 name = "syft-flwr"
-version = "0.2.0"
+version = "0.2.2"
 description = "syft_flwr is an open source framework that facilitate federated learning projects using Flower over the SyftBox protocol"
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
-    "syft-rds>=0.2.1",
-    "flwr[simulation]>=1.20.0",
+    "syft-rds>=0.2.2",
+    "flwr[simulation]==1.21.0",
     "flwr-datasets[vision]>=0.5.0",
     "loguru>=0.7.3",
     "safetensors>=0.6.2",
@@ -15,6 +15,10 @@ dependencies = [
     "tomli-w>=1.2.0",
 ]
 
+# [tool.uv.sources]
+# syft-rds = { path = "../syft-data-science" }  # for development
+
+
 [project.scripts]
 syft_flwr = "syft_flwr.cli:main"
 

{syft_flwr-0.2.0 → syft_flwr-0.2.2}/src/syft_flwr/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.2.0"
+__version__ = "0.2.2"
 
 from syft_flwr.bootstrap import bootstrap
 from syft_flwr.run import run

{syft_flwr-0.2.0 → syft_flwr-0.2.2}/src/syft_flwr/flower_client.py

@@ -6,14 +6,14 @@ from flwr.client import ClientApp
 from flwr.common import Context
 from flwr.common.constant import ErrorCode, MessageType
 from flwr.common.message import Error, Message
+from flwr.common.record import RecordDict
 from loguru import logger
 from syft_event import SyftEvents
 from syft_event.types import Request
 from typing_extensions import Optional, Union
 
-from syft_flwr.flwr_compatibility import RecordDict, create_flwr_message
 from syft_flwr.serde import bytes_to_flower_message, flower_message_to_bytes
-from syft_flwr.utils import setup_client
+from syft_flwr.utils import create_flwr_message, setup_client
 
 
 class MessageHandler:
@@ -50,10 +50,8 @@ class MessageHandler:
             content=RecordDict(),
             reply_to=message,
             message_type=message.metadata.message_type if message else MessageType.TASK,
-            src_node_id=message.metadata.dst_node_id if message else 0,
             dst_node_id=message.metadata.src_node_id if message else 0,
             group_id=message.metadata.group_id if message else "",
-            run_id=message.metadata.run_id if message else 0,
             error=error,
         )
         error_bytes = flower_message_to_bytes(error_reply)

{syft_flwr-0.2.0 → syft_flwr-0.2.2}/src/syft_flwr/grid.py

@@ -1,12 +1,15 @@
 import base64
 import os
+import random
 import time
 
 from flwr.common import ConfigRecord
 from flwr.common.constant import MessageType
 from flwr.common.message import Message
+from flwr.common.record import RecordDict
 from flwr.common.typing import Run
 from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
+from flwr.server.grid import Grid
 from loguru import logger
 from syft_core import Client
 from syft_crypto import EncryptedPayload, decrypt_message
@@ -14,20 +17,15 @@ from syft_rpc import SyftResponse, rpc, rpc_db
 from typing_extensions import Dict, Iterable, List, Optional, Tuple, cast
 
 from syft_flwr.consts import SYFT_FLWR_ENCRYPTION_ENABLED
-from syft_flwr.flwr_compatibility import (
-    Grid,
-    RecordDict,
-    check_reply_to_field,
-    create_flwr_message,
-)
 from syft_flwr.serde import bytes_to_flower_message, flower_message_to_bytes
-from syft_flwr.utils import str_to_int
+from syft_flwr.utils import check_reply_to_field, create_flwr_message, str_to_int
 
 # this is what superlink super node do
 AGGREGATOR_NODE_ID = 1
 
 # env vars
 SYFT_FLWR_MSG_TIMEOUT = "SYFT_FLWR_MSG_TIMEOUT"
+SYFT_FLWR_POLL_INTERVAL = "SYFT_FLWR_POLL_INTERVAL"
 
 
 class SyftGrid(Grid):
@@ -127,8 +125,6 @@ class SyftGrid(Grid):
             dst_node_id=dst_node_id,
             group_id=group_id,
             ttl=ttl,
-            run_id=cast(Run, self._run).run_id,
-            src_node_id=self.node.node_id,
         )
 
     def get_node_ids(self) -> list[int]:
@@ -318,27 +314,91 @@ class SyftGrid(Grid):
 
         return dest_datasite, url, msg_bytes
 
+    def _retry_with_backoff(
+        self,
+        func,
+        max_retries: int = 3,
+        initial_delay: float = 0.1,
+        context: str = "",
+        check_error=None,
+    ):
+        """Generic retry logic with exponential backoff and jitter.
+
+        Args:
+            func: Function to retry
+            max_retries: Maximum number of retry attempts
+            initial_delay: Initial delay in seconds
+            context: Context string for logging
+            check_error: Optional function to check if error is retryable
+
+        Returns:
+            Result of func if successful
+
+        Raises:
+            Last exception if all retries fail
+        """
+        for attempt in range(max_retries):
+            try:
+                return func()
+            except Exception as e:
+                is_retryable = check_error(e) if check_error else True
+                if is_retryable and attempt < max_retries - 1:
+                    jitter = random.uniform(0, 0.05)
+                    delay = initial_delay * (2**attempt) + jitter
+                    logger.debug(
+                        f"{context} failed (attempt {attempt + 1}/{max_retries}): {e}. "
+                        f"Retrying in {delay:.3f}s"
+                    )
+                    time.sleep(delay)
+                else:
+                    raise
+
+    def _save_future_with_retry(self, future, dest_datasite: str) -> bool:
+        """Save future to database with retry logic for database locks.
+
+        Returns:
+            True if saved successfully, False if failed after retries
+        """
+        try:
+            self._retry_with_backoff(
+                func=lambda: rpc_db.save_future(
+                    future=future, namespace=self.app_name, client=self._client
+                ),
+                context=f"Database save for {dest_datasite}",
+                check_error=lambda e: "database is locked" in str(e).lower(),
+            )
+            return True
+        except Exception as e:
+            logger.warning(
+                f"⚠️ Failed to save future to database for {dest_datasite}: {e}. "
+                f"Message sent but future not persisted."
+            )
+            return False
+
     def _send_encrypted_message(
         self, url: str, msg_bytes: bytes, dest_datasite: str, msg: Message
     ) -> Optional[str]:
         """Send an encrypted message and return future ID if successful."""
         try:
+            # Send encrypted message
             future = rpc.send(
                 url=url,
                 body=base64.b64encode(msg_bytes).decode("utf-8"),
                 client=self._client,
                 encrypt=True,
             )
+
             logger.debug(
                 f"🔐 Pushed ENCRYPTED message to {dest_datasite} at {url} "
                 f"with metadata {msg.metadata}; size {len(msg_bytes) / 1024 / 1024:.2f} MB"
             )
-            rpc_db.save_future(
-                future=future, namespace=self.app_name, client=self._client
-            )
+
+            # Save future to database (non-critical - log warning if fails)
+            self._save_future_with_retry(future, dest_datasite)
             return future.id
 
         except (KeyError, ValueError) as e:
+            # Encryption setup errors - don't retry or fallback
             error_type = (
                 "Encryption key" if isinstance(e, KeyError) else "Encryption parameter"
             )
@@ -349,6 +409,7 @@ class SyftGrid(Grid):
             return None
 
         except Exception as e:
+            # Other errors - fallback to unencrypted
             logger.warning(
                 f"⚠️ Encryption failed for {dest_datasite}: {e}. "
                 f"Falling back to unencrypted transmission"
@@ -382,6 +443,9 @@ class SyftGrid(Grid):
         responses = {}
         pending_ids = msg_ids.copy()
 
+        # Get polling interval from environment or use default
+        poll_interval = float(os.environ.get(SYFT_FLWR_POLL_INTERVAL, "3"))
+
         while pending_ids and (timeout is None or time.time() < end_time):
             # Pull available messages
             batch = self.pull_messages(pending_ids)
@@ -389,7 +453,7 @@ class SyftGrid(Grid):
             pending_ids.difference_update(batch.keys())
 
             if pending_ids:
-                time.sleep(3)  # Polling interval
+                time.sleep(poll_interval)  # Configurable polling interval
 
         # Log any missing responses
         if pending_ids:

{syft_flwr-0.2.0 → syft_flwr-0.2.2}/src/syft_flwr/run.py

@@ -5,11 +5,12 @@ from uuid import uuid4
 from flwr.client.client_app import LoadClientAppError
 from flwr.common import Context
 from flwr.common.object_ref import load_app
+from flwr.common.record import RecordDict
 from flwr.server.server_app import LoadServerAppError
+
 from syft_flwr.config import load_flwr_pyproject
 from syft_flwr.flower_client import syftbox_flwr_client
 from syft_flwr.flower_server import syftbox_flwr_server
-from syft_flwr.flwr_compatibility import RecordDict
 from syft_flwr.run_simulation import run
 
 __all__ = ["syftbox_run_flwr_client", "syftbox_run_flwr_server", "run"]

{syft_flwr-0.2.0 → syft_flwr-0.2.2}/src/syft_flwr/utils.py

@@ -3,10 +3,13 @@ import re
 import zlib
 from pathlib import Path
 
+from flwr.common import Metadata
+from flwr.common.message import Error, Message
+from flwr.common.record import RecordDict
 from loguru import logger
 from syft_core import Client, SyftClientConfig
 from syft_crypto.x3dh_bootstrap import ensure_bootstrap
-from typing_extensions import Tuple
+from typing_extensions import Optional, Tuple
 
 from syft_flwr.consts import SYFT_FLWR_ENCRYPTION_ENABLED
 
@@ -79,3 +82,34 @@ def setup_client(app_name: str) -> Tuple[Client, bool, str]:
         )
 
     return client, encryption_enabled, f"flwr/{app_name}"
+
+
+def check_reply_to_field(metadata: Metadata) -> bool:
+    """Check if reply_to field is empty (Flower 1.17+ format)."""
+    return metadata.reply_to_message_id == ""
+
+
+def create_flwr_message(
+    content: RecordDict,
+    message_type: str,
+    dst_node_id: int,
+    group_id: str,
+    ttl: Optional[float] = None,
+    error: Optional[Error] = None,
+    reply_to: Optional[Message] = None,
+) -> Message:
+    """Create a Flower message (requires Flower >= 1.17)."""
+    if reply_to is not None:
+        if error is not None:
+            return Message(reply_to=reply_to, error=error)
+        return Message(content=content, reply_to=reply_to)
+    else:
+        if error is not None:
+            raise ValueError("Error and reply_to cannot both be None")
+        return Message(
+            content=content,
+            dst_node_id=dst_node_id,
+            message_type=message_type,
+            ttl=ttl,
+            group_id=group_id,
+        )

syft_flwr-0.2.0/src/syft_flwr/flwr_compatibility.py

@@ -1,121 +0,0 @@
-import flwr
-from flwr.common import Metadata
-from flwr.common.message import Error, Message
-from packaging.version import Version
-from typing_extensions import Optional
-
-
-def flwr_later_than_1_17():
-    return Version(flwr.__version__) >= Version("1.17.0")
-
-
-# Version-dependent imports
-if flwr_later_than_1_17():
-    from flwr.common.record import RecordDict
-    from flwr.server.grid import Grid
-else:
-    from flwr.common.record import RecordSet as RecordDict
-    from flwr.server.driver import Driver as Grid
-
-
-__all__ = ["Grid", "RecordDict"]
-
-
-def check_reply_to_field(metadata: Metadata) -> bool:
-    """Check if reply_to field is empty based on Flower version."""
-    if flwr_later_than_1_17():
-        return metadata.reply_to_message_id == ""
-    else:
-        return metadata.reply_to_message == ""
-
-
-def create_flwr_message(
-    content: RecordDict,
-    message_type: str,
-    src_node_id: int,
-    dst_node_id: int,
-    group_id: str,
-    run_id: int,
-    ttl: Optional[float] = None,
-    error: Optional[Error] = None,
-    reply_to: Optional[Message] = None,
-) -> Message:
-    """Create a Flower message with version-compatible parameters."""
-    if flwr_later_than_1_17():
-        return _create_message_v1_17_plus(
-            content,
-            message_type,
-            dst_node_id,
-            group_id,
-            ttl,
-            error,
-            reply_to,
-        )
-    else:
-        return _create_message_pre_v1_17(
-            content,
-            message_type,
-            src_node_id,
-            dst_node_id,
-            group_id,
-            run_id,
-            ttl,
-            error,
-        )
-
-
-def _create_message_v1_17_plus(
-    content: RecordDict,
-    message_type: str,
-    dst_node_id: int,
-    group_id: str,
-    ttl: Optional[float],
-    error: Optional[Error],
-    reply_to: Optional[Message],
-) -> Message:
-    """Create message for Flower version 1.17+."""
-    if reply_to is not None:
-        if error is not None:
-            return Message(reply_to=reply_to, error=error)
-        return Message(content=content, reply_to=reply_to)
-    else:
-        if error is not None:
-            raise ValueError("Error and reply_to cannot both be None")
-        return Message(
-            content=content,
-            dst_node_id=dst_node_id,
-            message_type=message_type,
-            ttl=ttl,
-            group_id=group_id,
-        )
-
-
-def _create_message_pre_v1_17(
-    content: RecordDict,
-    message_type: str,
-    src_node_id: int,
-    dst_node_id: int,
-    group_id: str,
-    run_id: int,
-    ttl: Optional[float],
-    error: Optional[Error],
-) -> Message:
-    """Create message for Flower versions before 1.17."""
-    from flwr.common import DEFAULT_TTL
-
-    ttl_ = DEFAULT_TTL if ttl is None else ttl
-    metadata = Metadata(
-        run_id=run_id,
-        message_id="",  # Will be set when saving to file
-        src_node_id=src_node_id,
-        dst_node_id=dst_node_id,
-        reply_to_message="",
-        group_id=group_id,
-        ttl=ttl_,
-        message_type=message_type,
-    )
-
-    if error is not None:
-        return Message(metadata=metadata, error=error)
-    else:
-        return Message(metadata=metadata, content=content)