flwr-nightly 1.8.0.dev20240327__py3-none-any.whl → 1.8.0.dev20240328__py3-none-any.whl

flwr/client/app.py

@@ -14,11 +14,10 @@
 # ==============================================================================
 """Flower client app."""
 
-
 import argparse
 import sys
 import time
-from logging import DEBUG, INFO, WARN
+from logging import DEBUG, ERROR, INFO, WARN
 from pathlib import Path
 from typing import Callable, ContextManager, Optional, Tuple, Type, Union
 
@@ -38,6 +37,7 @@ from flwr.common.constant import (
 )
 from flwr.common.exit_handlers import register_exit_handlers
 from flwr.common.logger import log, warn_deprecated_feature, warn_experimental_feature
+from flwr.common.message import Error
 from flwr.common.object_ref import load_app, validate
 from flwr.common.retry_invoker import RetryInvoker, exponential
 
@@ -482,32 +482,43 @@ def _start_client_internal(
                 # Retrieve context for this run
                 context = node_state.retrieve_context(run_id=message.metadata.run_id)
 
-                # Load ClientApp instance
-                client_app: ClientApp = load_client_app_fn()
+                # Create an error reply message that will never be used to prevent
+                # the used-before-assignment linting error
+                reply_message = message.create_error_reply(
+                    error=Error(code=0, reason="Unknown")
+                )
 
-                # Handle task message
-                out_message = client_app(message=message, context=context)
+                # Handle app loading and task message
+                try:
+                    # Load ClientApp instance
+                    client_app: ClientApp = load_client_app_fn()
 
-                # Update node state
-                node_state.update_context(
-                    run_id=message.metadata.run_id,
-                    context=context,
-                )
+                    reply_message = client_app(message=message, context=context)
+                    # Update node state
+                    node_state.update_context(
+                        run_id=message.metadata.run_id,
+                        context=context,
+                    )
+                except Exception as ex:  # pylint: disable=broad-exception-caught
+                    log(ERROR, "ClientApp raised an exception", exc_info=ex)
+
+                    # Legacy grpc-bidi
+                    if transport in ["grpc-bidi", None]:
+                        # Raise exception, crash process
+                        raise ex
+
+                    # Don't update/change NodeState
+
+                    # Create error message
+                    # Reason example: "<class 'ZeroDivisionError'>:<'division by zero'>"
+                    reason = str(type(ex)) + ":<'" + str(ex) + "'>"
+                    reply_message = message.create_error_reply(
+                        error=Error(code=0, reason=reason)
+                    )
 
                 # Send
-                send(out_message)
-                log(
-                    INFO,
-                    "[RUN %s, ROUND %s]",
-                    out_message.metadata.run_id,
-                    out_message.metadata.group_id,
-                )
-                log(
-                    INFO,
-                    "Sent: %s reply to message %s",
-                    out_message.metadata.message_type,
-                    message.metadata.message_id,
-                )
+                send(reply_message)
+                log(INFO, "Sent reply")
 
             # Unregister node
             if delete_node is not None:

flwr/common/message.py

@@ -297,22 +297,33 @@ class Message:
             partition_id=self.metadata.partition_id,
         )
 
-    def create_error_reply(
-        self,
-        error: Error,
-        ttl: float,
-    ) -> Message:
+    def create_error_reply(self, error: Error, ttl: float | None = None) -> Message:
         """Construct a reply message indicating an error happened.
 
         Parameters
         ----------
         error : Error
             The error that was encountered.
-        ttl : float
-            Time-to-live for this message in seconds.
+        ttl : Optional[float] (default: None)
+            Time-to-live for this message in seconds. If unset, it will be set based
+            on the remaining time for the received message before it expires. This
+            follows the equation:
+
+            ttl = msg.meta.ttl - (reply.meta.created_at - msg.meta.created_at)
         """
+        # If no TTL passed, use default for message creation (will update after
+        # message creation)
+        ttl_ = DEFAULT_TTL if ttl is None else ttl
         # Create reply with error
-        message = Message(metadata=self._create_reply_metadata(ttl), error=error)
+        message = Message(metadata=self._create_reply_metadata(ttl_), error=error)
+
+        if ttl is None:
+            # Set TTL equal to the remaining time for the received message to expire
+            ttl = self.metadata.ttl - (
+                message.metadata.created_at - self.metadata.created_at
+            )
+            message.metadata.ttl = ttl
+
         return message
 
     def create_reply(self, content: RecordSet, ttl: float | None = None) -> Message:
@@ -327,18 +338,31 @@ class Message:
         content : RecordSet
             The content for the reply message.
         ttl : Optional[float] (default: None)
-            Time-to-live for this message in seconds. If unset, it will use
-            the `common.DEFAULT_TTL` value.
+            Time-to-live for this message in seconds. If unset, it will be set based
+            on the remaining time for the received message before it expires. This
+            follows the equation:
+
+            ttl = msg.meta.ttl - (reply.meta.created_at - msg.meta.created_at)
 
         Returns
         -------
         Message
             A new `Message` instance representing the reply.
         """
-        if ttl is None:
-            ttl = DEFAULT_TTL
+        # If no TTL passed, use default for message creation (will update after
+        # message creation)
+        ttl_ = DEFAULT_TTL if ttl is None else ttl
 
-        return Message(
-            metadata=self._create_reply_metadata(ttl),
+        message = Message(
+            metadata=self._create_reply_metadata(ttl_),
             content=content,
         )
+
+        if ttl is None:
+            # Set TTL equal to the remaining time for the received message to expire
+            ttl = self.metadata.ttl - (
+                message.metadata.created_at - self.metadata.created_at
+            )
+            message.metadata.ttl = ttl
+
+        return message

flwr/server/compat/driver_client_proxy.py

@@ -170,8 +170,24 @@ class DriverClientProxy(ClientProxy):
             )
             if len(task_res_list) == 1:
                 task_res = task_res_list[0]
+
+                # This will raise an Exception if task_res carries an `error`
+                validate_task_res(task_res=task_res)
+
                 return serde.recordset_from_proto(task_res.task.recordset)
 
             if timeout is not None and time.time() > start_time + timeout:
                 raise RuntimeError("Timeout reached")
             time.sleep(SLEEP_TIME)
+
+
+def validate_task_res(
+    task_res: task_pb2.TaskRes,  # pylint: disable=E1101
+) -> None:
+    """Validate if a TaskRes is empty or not."""
+    if not task_res.HasField("task"):
+        raise ValueError("Invalid TaskRes, field `task` missing")
+    if task_res.task.HasField("error"):
+        raise ValueError("Exception during client-side task execution")
+    if not task_res.task.HasField("recordset"):
+        raise ValueError("Invalid TaskRes, both `recordset` and `error` are missing")

flwr/server/superlink/fleet/vce/backend/raybackend.py

@@ -20,7 +20,7 @@ from typing import Callable, Dict, List, Tuple, Union
 
 import ray
 
-from flwr.client.client_app import ClientApp, LoadClientAppError
+from flwr.client.client_app import ClientApp
 from flwr.common.context import Context
 from flwr.common.logger import log
 from flwr.common.message import Message
@@ -151,7 +151,6 @@ class RayBackend(Backend):
             )
 
             await future
-
             # Fetch result
             (
                 out_mssg,
@@ -160,13 +159,15 @@ class RayBackend(Backend):
 
             return out_mssg, updated_context
 
-        except LoadClientAppError as load_ex:
+        except Exception as ex:
             log(
                 ERROR,
                 "An exception was raised when processing a message by %s",
                 self.__class__.__name__,
             )
-            raise load_ex
+            # add actor back into pool
+            await self.pool.add_actor_back_to_pool(future)
+            raise ex
 
     async def terminate(self) -> None:
         """Terminate all actors in actor pool."""

flwr/server/superlink/fleet/vce/vce_api.py

@@ -14,9 +14,10 @@
 # ==============================================================================
 """Fleet Simulation Engine API."""
 
-
 import asyncio
 import json
+import sys
+import time
 import traceback
 from logging import DEBUG, ERROR, INFO, WARN
 from typing import Callable, Dict, List, Optional
@@ -24,6 +25,7 @@ from typing import Callable, Dict, List, Optional
 from flwr.client.client_app import ClientApp, LoadClientAppError
 from flwr.client.node_state import NodeState
 from flwr.common.logger import log
+from flwr.common.message import Error
 from flwr.common.object_ref import load_app
 from flwr.common.serde import message_from_taskins, message_to_taskres
 from flwr.proto.task_pb2 import TaskIns  # pylint: disable=E0611
@@ -59,6 +61,7 @@ async def worker(
     """Get TaskIns from queue and pass it to an actor in the pool to execute it."""
     state = state_factory.state()
     while True:
+        out_mssg = None
         try:
             task_ins: TaskIns = await queue.get()
             node_id = task_ins.task.consumer.node_id
@@ -82,24 +85,25 @@ async def worker(
                 task_ins.run_id, context=updated_context
             )
 
-            # Convert to TaskRes
-            task_res = message_to_taskres(out_mssg)
-            # Store TaskRes in state
-            state.store_task_res(task_res)
-
         except asyncio.CancelledError as e:
-            log(DEBUG, "Async worker: %s", e)
+            log(DEBUG, "Terminating async worker: %s", e)
             break
 
-        except LoadClientAppError as app_ex:
-            log(ERROR, "Async worker: %s", app_ex)
-            log(ERROR, traceback.format_exc())
-            raise
-
+        # Exceptions aren't raised but reported as an error message
         except Exception as ex:  # pylint: disable=broad-exception-caught
             log(ERROR, ex)
             log(ERROR, traceback.format_exc())
-            break
+            reason = str(type(ex)) + ":<'" + str(ex) + "'>"
+            error = Error(code=0, reason=reason)
+            out_mssg = message.create_error_reply(error=error)
+
+        finally:
+            if out_mssg:
+                # Convert to TaskRes
+                task_res = message_to_taskres(out_mssg)
+                # Store TaskRes in state
+                task_res.task.pushed_at = time.time()
+                state.store_task_res(task_res)
 
 
 async def add_taskins_to_queue(
@@ -218,7 +222,7 @@ async def run(
         await backend.terminate()
 
 
-# pylint: disable=too-many-arguments,unused-argument,too-many-locals
+# pylint: disable=too-many-arguments,unused-argument,too-many-locals,too-many-branches
 def start_vce(
     backend_name: str,
     backend_config_json_stream: str,
@@ -300,12 +304,14 @@ def start_vce(
         """Instantiate a Backend."""
         return backend_type(backend_config, work_dir=app_dir)
 
-    log(INFO, "client_app_attr = %s", client_app_attr)
-
     # Load ClientApp if needed
     def _load() -> ClientApp:
 
         if client_app_attr:
+
+            if app_dir is not None:
+                sys.path.insert(0, app_dir)
+
             app: ClientApp = load_app(client_app_attr, LoadClientAppError)
 
             if not isinstance(app, ClientApp):
@@ -319,13 +325,23 @@ def start_vce(
 
     app_fn = _load
 
-    asyncio.run(
-        run(
-            app_fn,
-            backend_fn,
-            nodes_mapping,
-            state_factory,
-            node_states,
-            f_stop,
+    try:
+        # Test if ClientApp can be loaded
+        _ = app_fn()
+
+        # Run main simulation loop
+        asyncio.run(
+            run(
+                app_fn,
+                backend_fn,
+                nodes_mapping,
+                state_factory,
+                node_states,
+                f_stop,
+            )
         )
-    )
+    except LoadClientAppError as loadapp_ex:
+        f_stop.set()  # set termination event
+        raise loadapp_ex
+    except Exception as ex:
+        raise ex

flwr/simulation/ray_transport/ray_actor.py

@@ -493,13 +493,17 @@ class BasicActorPool:
         self._future_to_actor[future] = actor
         return future
 
+    async def add_actor_back_to_pool(self, future: Any) -> None:
+        """Ad actor assigned to run future back into the pool."""
+        actor = self._future_to_actor.pop(future)
+        await self.pool.put(actor)
+
     async def fetch_result_and_return_actor_to_pool(
         self, future: Any
     ) -> Tuple[Message, Context]:
         """Pull result given a future and add actor back to pool."""
         # Get actor that ran job
-        actor = self._future_to_actor.pop(future)
-        await self.pool.put(actor)
+        await self.add_actor_back_to_pool(future)
         # Retrieve result for object store
         # Instead of doing ray.get(future) we await it
         _, out_mssg, updated_context = await future

{flwr_nightly-1.8.0.dev20240327.dist-info → flwr_nightly-1.8.0.dev20240328.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.8.0.dev20240327
+Version: 1.8.0.dev20240328
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0

{flwr_nightly-1.8.0.dev20240327.dist-info → flwr_nightly-1.8.0.dev20240328.dist-info}/RECORD

@@ -28,7 +28,7 @@ flwr/cli/run/__init__.py,sha256=oCd6HmQDx-sqver1gecgx-uMA38BLTSiiKpl7RGNceg,789
 flwr/cli/run/run.py,sha256=C7Yh-Y0f64PEabb9733jBKIhhOUFpcRmCZJIDtv-NG8,2329
 flwr/cli/utils.py,sha256=_V2BlFVNNG2naZrq227fZ8o4TxBN_hB-4fQsen9uQoo,2300
 flwr/client/__init__.py,sha256=futk_IdY_N1h8BTve4Iru51bxm7H1gv58ZPIXWi5XUA,1187
-flwr/client/app.py,sha256=xCCpP-fMEFdTEaSWOP93JPIDfjRhx5Z1uI1h6YlJteo,24784
+flwr/client/app.py,sha256=bRrLIfn3MIZKfsVq8AlgYmhpI08UTioaOlyx6YLaLR4,25536
 flwr/client/client.py,sha256=Vp9UkOkoHdNfn6iMYZsj_5m_GICiFfUlKEVaLad-YhM,8183
 flwr/client/client_app.py,sha256=30Tl_AOEi4CE8wEQbKJ3tWg4GfbsSoV1Ztc8iWE0ge4,8047
 flwr/client/dpfedavg_numpy_client.py,sha256=9Tnig4iml2J88HBKNahegjXjbfvIQyBtaIQaqjbeqsA,7435
@@ -64,7 +64,7 @@ flwr/common/dp.py,sha256=Hc3lLHihjexbJaD_ft31gdv9XRcwOTgDBwJzICuok3A,2004
 flwr/common/exit_handlers.py,sha256=2Nt0wLhc17KQQsLPFSRAjjhUiEFfJK6tNozdGiIY4Fs,2812
 flwr/common/grpc.py,sha256=HimjpTtIY3Vfqtlq3u-CYWjqAl9rSn0uo3A8JjhUmwQ,2273
 flwr/common/logger.py,sha256=Plhm9fsi4ewb90eGALQZ9xBkR0cGEsckX5RLSMEaS3M,6118
-flwr/common/message.py,sha256=OsYdWVt5ovWTxiGyC3nav61GknoGUyH_p-zR6J-L0Dk,10532
+flwr/common/message.py,sha256=vgFSCOkPbl60iS4-XQJ8-rHL54MvNc2AwNSSxVl6qYY,11773
 flwr/common/object_ref.py,sha256=ELoUCAFO-vbjJC41CGpa-WBG2SLYe3ErW-d9YCG3zqA,4961
 flwr/common/parameter.py,sha256=-bFAUayToYDF50FZGrBC1hQYJCQDtB2bbr3ZuVLMtdE,2095
 flwr/common/pyproject.py,sha256=EI_ovbCHGmhYrdPx0RSDi5EkFZFof-8m1PA54c0ZTjc,1385
@@ -126,7 +126,7 @@ flwr/server/client_proxy.py,sha256=4G-oTwhb45sfWLx2uZdcXD98IZwdTS6F88xe3akCdUg,2
 flwr/server/compat/__init__.py,sha256=VxnJtJyOjNFQXMNi9hIuzNlZM5n0Hj1p3aq_Pm2udw4,892
 flwr/server/compat/app.py,sha256=3Skh76Rg80B4oME1dJOhZvn9eTfVmTNIQ0jCiZ6CzeQ,5271
 flwr/server/compat/app_utils.py,sha256=-Ey5fyRpovmp4nHglVbliITcbxzxX_0qdtZwwfMS4ZI,3450
-flwr/server/compat/driver_client_proxy.py,sha256=otGgR_0KOadja4s0GO3zTAO0DoDGzzt6tUq0OHFS8gI,6719
+flwr/server/compat/driver_client_proxy.py,sha256=QWLl5YJwI6NVADwjQGQJqkLtCfPNT-aRH0NF9yeGEnA,7344
 flwr/server/compat/legacy_context.py,sha256=D2s7PvQoDnTexuRmf1uG9Von7GUj4Qqyr7qLklSlKAM,1766
 flwr/server/criterion.py,sha256=ypbAexbztzGUxNen9RCHF91QeqiEQix4t4Ih3E-42MM,1061
 flwr/server/driver/__init__.py,sha256=yYyVX1FcDiDFM6rw0-DSZpuRy0EoWRfG9puwlQUswFA,820
@@ -180,8 +180,8 @@ flwr/server/superlink/fleet/rest_rere/rest_api.py,sha256=7JCs7NW4Qq8W5QhXxqsQNFi
 flwr/server/superlink/fleet/vce/__init__.py,sha256=36MHKiefnJeyjwMQzVUK4m06Ojon3WDcwZGQsAcyVhQ,783
 flwr/server/superlink/fleet/vce/backend/__init__.py,sha256=oBIzmnrSSRvH_H0vRGEGWhWzQQwqe3zn6e13RsNwlIY,1466
 flwr/server/superlink/fleet/vce/backend/backend.py,sha256=LJsKl7oixVvptcG98Rd9ejJycNWcEVB0ODvSreLGp-A,2260
-flwr/server/superlink/fleet/vce/backend/raybackend.py,sha256=BYgzVH1uz8nk6mOP6GhgSxjrdCe7xtkzb7nhPbKStFM,6317
-flwr/server/superlink/fleet/vce/vce_api.py,sha256=Yq4i9fduafnoWSHCLn0mmkCTS9oZqwycH8gbKa4bPXo,11168
+flwr/server/superlink/fleet/vce/backend/raybackend.py,sha256=TaT2EpbVEsIY0EDzF8obadyZaSXjD38TFGdDPI-ytD0,6375
+flwr/server/superlink/fleet/vce/vce_api.py,sha256=EV4ISvHZPucVLD3lYVFF5fQ4yyxmoaoZMdp_1B2k6J8,11789
 flwr/server/superlink/state/__init__.py,sha256=ij-7Ms-hyordQdRmGQxY1-nVa4OhixJ0jr7_YDkys0s,1003
 flwr/server/superlink/state/in_memory_state.py,sha256=XfdCGRzFut9xlf7AsDAhhAmBw-nKDBjmPWAI--espj0,8707
 flwr/server/superlink/state/sqlite_state.py,sha256=1SR6Zz6ud0tSSx940gTfa0vct_GH2n0cX_vnhoAEMlQ,22005
@@ -200,12 +200,12 @@ flwr/server/workflow/secure_aggregation/secaggplus_workflow.py,sha256=3TjJdhYA4x
 flwr/simulation/__init__.py,sha256=hpoKzdovrH0_Cf8HIcXxQxyUUb3BiSk-WUNLf5STHcc,1400
 flwr/simulation/app.py,sha256=WqJxdXTEuehwMW605p5NMmvBbKYx5tuqnV3Mp7jSWXM,13904
 flwr/simulation/ray_transport/__init__.py,sha256=FsaAnzC4cw4DqoouBCix6496k29jACkfeIam55BvW9g,734
-flwr/simulation/ray_transport/ray_actor.py,sha256=zRETW_xuCAOLRFaYnQ-q3IBSz0LIv_0RifGuhgWaYOg,19872
+flwr/simulation/ray_transport/ray_actor.py,sha256=OWjgYW--fswkEDqTP9L_cZblBeUVL59vNz5gvzPAHFk,20054
 flwr/simulation/ray_transport/ray_client_proxy.py,sha256=oDu4sEPIOu39vrNi-fqDAe10xtNUXMO49bM2RWfRcyw,6738
 flwr/simulation/ray_transport/utils.py,sha256=TYdtfg1P9VfTdLMOJlifInGpxWHYs9UfUqIv2wfkRLA,2392
 flwr/simulation/run_simulation.py,sha256=HiIH6aa_v56NfKQN5ZBd94NyVfaZNyFs43_kItYsQXU,15685
-flwr_nightly-1.8.0.dev20240327.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-flwr_nightly-1.8.0.dev20240327.dist-info/METADATA,sha256=k34dkRQ99eG3DkdjPCB3bEEllC1gMNxnYJBXZtHv6P4,15257
-flwr_nightly-1.8.0.dev20240327.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-flwr_nightly-1.8.0.dev20240327.dist-info/entry_points.txt,sha256=utu2wybGyYJSTtsB2ktY_gmy-XtMFo9EFZdishX0zR4,320
-flwr_nightly-1.8.0.dev20240327.dist-info/RECORD,,
+flwr_nightly-1.8.0.dev20240328.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+flwr_nightly-1.8.0.dev20240328.dist-info/METADATA,sha256=8FreyB01iNsm1LCt2zh74ZE46oAFcKAieUIrZQEtTgI,15257
+flwr_nightly-1.8.0.dev20240328.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+flwr_nightly-1.8.0.dev20240328.dist-info/entry_points.txt,sha256=utu2wybGyYJSTtsB2ktY_gmy-XtMFo9EFZdishX0zR4,320
+flwr_nightly-1.8.0.dev20240328.dist-info/RECORD,,