hypha-rpc 0.20.85__tar.gz → 0.20.86__tar.gz

{hypha_rpc-0.20.85 → hypha_rpc-0.20.86}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hypha_rpc
-Version: 0.20.85
+Version: 0.20.86
 Summary: Hypha RPC client for connecting to Hypha server for data management and AI model serving
 Author-email: Wei Ouyang <oeway007@gmail.com>
 Requires-Python: >=3.9

hypha_rpc-0.20.86/hypha_rpc/VERSION

@@ -0,0 +1,3 @@
+{
+  "version": "0.20.86"
+}

{hypha_rpc-0.20.85 → hypha_rpc-0.20.86}/hypha_rpc/__init__.py

@@ -7,7 +7,7 @@ import asyncio
 import sys
 
 from .rpc import RPC
-from .utils import ObjectProxy
+from .utils import ObjectProxy, run_in_executor
 from .sync import connect_to_server as connect_to_server_sync
 from .sync import get_remote_service as get_remote_service_sync
 from .sync import get_rtc_service as get_rtc_service_sync
@@ -127,4 +127,5 @@ __all__ = [
     "register_rtc_service_sync",
     "get_rtc_service_sync",
     "setup_local_client",
+    "run_in_executor",
 ]

{hypha_rpc-0.20.85 → hypha_rpc-0.20.86}/hypha_rpc/rpc.py

@@ -669,14 +669,25 @@ class RPC(MessageEmitter):
                         registered_count = 0
                         failed_services = []
 
+                        # Use timeout for service registration to prevent hanging
+                        service_registration_timeout = self._method_timeout or 30
+
                         for service in list(self._services.values()):
                             try:
                                 service_info = self._extract_service_info(service)
-                                await manager.register_service(service_info)
+                                await asyncio.wait_for(
+                                    manager.register_service(service_info),
+                                    timeout=service_registration_timeout
+                                )
                                 registered_count += 1
                                 logger.debug(
                                     f"Successfully registered service: {service.get('id', 'unknown')}"
                                 )
+                            except asyncio.TimeoutError:
+                                failed_services.append(service.get("id", "unknown"))
+                                logger.error(
+                                    f"Timeout registering service {service.get('id', 'unknown')}"
+                                )
                             except Exception as service_error:
                                 failed_services.append(service.get("id", "unknown"))
                                 logger.error(
@@ -707,25 +718,29 @@ class RPC(MessageEmitter):
                             manager_dict = ObjectProxy.toDict(manager)
                             if "subscribe" in manager_dict:
                                 logger.debug("Subscribing to client_disconnected events")
-                                
+
                                 async def handle_client_disconnected(event):
                                     client_id = event.get("client")
                                     if client_id:
                                         logger.debug(f"Client {client_id} disconnected, cleaning up sessions")
                                         await self._handle_client_disconnected(client_id)
-                                
-                                # Subscribe to the event topic first
-                                self._client_disconnected_subscription = await manager.subscribe([
-                                    "client_disconnected"
-                                ])
-                                
+
+                                # Subscribe to the event topic first with timeout
+                                self._client_disconnected_subscription = await asyncio.wait_for(
+                                    manager.subscribe(["client_disconnected"]),
+                                    timeout=service_registration_timeout
+                                )
+
                                 # Then register the local event handler
                                 self.on("client_disconnected", handle_client_disconnected)
-                                
+
                                 logger.debug("Successfully subscribed to client_disconnected events")
                             else:
                                 logger.debug("Manager does not support subscribe method, skipping client_disconnected handling")
                                 self._client_disconnected_subscription = None
+                        except asyncio.TimeoutError:
+                            logger.warning("Timeout subscribing to client_disconnected events")
+                            self._client_disconnected_subscription = None
                         except Exception as subscribe_error:
                             logger.warning(f"Failed to subscribe to client_disconnected events: {subscribe_error}")
                             self._client_disconnected_subscription = None
@@ -990,29 +1005,14 @@ class RPC(MessageEmitter):
             except Exception as e:
                 logger.debug(f"Error removing background task: {e}")
         
-        # Unsubscribe from client_disconnected events if subscribed
+        # Remove the local event handler for client_disconnected
+        # Note: Actual unsubscription from server is done in async disconnect() method
         if hasattr(self, '_client_disconnected_subscription') and self._client_disconnected_subscription:
             try:
-                # Get the manager service to unsubscribe (non-blocking)
-                if self._connection and self._connection.manager_id:
-                    async def unsubscribe_async():
-                        try:
-                            manager = await self.get_remote_service(f"*/{self._connection.manager_id}")
-                            if hasattr(manager, 'unsubscribe') and callable(manager.unsubscribe):
-                                if asyncio.iscoroutinefunction(manager.unsubscribe):
-                                    await manager.unsubscribe("client_disconnected")
-                                else:
-                                    manager.unsubscribe("client_disconnected")
-                        except Exception as e:
-                            logger.debug(f"Error unsubscribing from client_disconnected: {e}")
-                    
-                    # Create task to run in background
-                    asyncio.create_task(unsubscribe_async())
-                    
                 # Remove the local event handler
                 self.off("client_disconnected")
             except Exception as e:
-                logger.debug(f"Error unsubscribing from client_disconnected: {e}")
+                logger.debug(f"Error removing client_disconnected handler: {e}")
         
         # Clear connection reference to break circular references
         if hasattr(self, '_connection'):
@@ -1026,10 +1026,36 @@ class RPC(MessageEmitter):
 
     async def disconnect(self):
         """Disconnect."""
-        # Store connection reference before closing
+        # Store connection reference before closing for unsubscribe
         connection = getattr(self, '_connection', None)
+        manager_id = connection.manager_id if connection else None
+
+        # Unsubscribe from client_disconnected events before closing
+        if hasattr(self, '_client_disconnected_subscription') and self._client_disconnected_subscription:
+            try:
+                if connection and manager_id:
+                    manager = await asyncio.wait_for(
+                        self.get_remote_service(f"*/{manager_id}"),
+                        timeout=5.0
+                    )
+                    if hasattr(manager, 'unsubscribe') and callable(manager.unsubscribe):
+                        if asyncio.iscoroutinefunction(manager.unsubscribe):
+                            await asyncio.wait_for(
+                                manager.unsubscribe("client_disconnected"),
+                                timeout=5.0
+                            )
+                        else:
+                            manager.unsubscribe("client_disconnected")
+                    logger.debug("Successfully unsubscribed from client_disconnected events")
+            except asyncio.TimeoutError:
+                logger.debug("Timeout unsubscribing from client_disconnected events")
+            except Exception as e:
+                logger.debug(f"Error unsubscribing from client_disconnected: {e}")
+            finally:
+                self._client_disconnected_subscription = None
+
         self.close()
-        
+
         # Disconnect the underlying connection if it exists
         if connection:
             try:

{hypha_rpc-0.20.85 → hypha_rpc-0.20.86}/hypha_rpc/utils/__init__.py

@@ -74,6 +74,41 @@ def safe_create_future():
             return asyncio.Future()
 
 
+async def run_in_executor(func, *args, executor=None, **kwargs):
+    """
+    Run a synchronous CPU-bound function in an executor to avoid blocking the event loop.
+
+    This is critical for maintaining RPC stability when services perform CPU-intensive
+    operations. Blocking the event loop can prevent:
+    - Heartbeat messages from being sent/received
+    - Reconnection logic from executing
+    - Other concurrent RPC calls from progressing
+
+    Example usage:
+        # Instead of:
+        result = cpu_intensive_function(data)
+
+        # Use:
+        from hypha_rpc.utils import run_in_executor
+        result = await run_in_executor(cpu_intensive_function, data)
+
+    Args:
+        func: The synchronous function to run
+        *args: Positional arguments to pass to the function
+        executor: Optional concurrent.futures.Executor. If None, uses the default
+                  ThreadPoolExecutor. For truly CPU-bound tasks, consider using
+                  a ProcessPoolExecutor.
+        **kwargs: Keyword arguments to pass to the function
+
+    Returns:
+        The result of calling func(*args, **kwargs)
+    """
+    loop = asyncio.get_running_loop()
+    if kwargs:
+        func = partial(func, **kwargs)
+    return await loop.run_in_executor(executor, func, *args)
+
+
 # The following code adopted from the munch library,
 # By Copyright (c) 2010 David Schoonover
 # We changed the way the keys being generated to cope with

{hypha_rpc-0.20.85 → hypha_rpc-0.20.86}/hypha_rpc/websocket_client.py

@@ -219,6 +219,12 @@ class WebsocketRPCConnection:
         except asyncio.CancelledError:
             # Task was cancelled, cleanup or exit gracefully
             logger.info("Refresh token task was cancelled.")
+        except RuntimeError as e:
+            # Handle event loop closed error gracefully
+            if "Event loop is closed" in str(e) or "cannot schedule new futures" in str(e):
+                logger.debug("Event loop closed during refresh token task")
+            else:
+                logger.error(f"RuntimeError in refresh token task: {e}")
         except Exception as exp:
             logger.error(f"Failed to send refresh token: {exp}")
 
@@ -427,7 +433,13 @@ class WebsocketRPCConnection:
                                 )
                                 break
                             except ConnectionAbortedError as e:
-                                logger.warning("Server refuse to reconnect: %s", e)
+                                logger.warning("Server refused to reconnect: %s", e)
+                                # Mark as closed and notify the application
+                                self._closed = True
+                                if self._handle_disconnected:
+                                    self._handle_disconnected(
+                                        f"Server refused reconnection: {e}"
+                                    )
                                 break
                             except (ConnectionRefusedError, OSError) as e:
                                 # Network-related errors that might be temporary
@@ -483,19 +495,18 @@ class WebsocketRPCConnection:
 
                         if retry >= MAX_RETRY and not self._closed:
                             logger.error(
-                                f"Failed to reconnect after {MAX_RETRY} attempts, giving up. Exiting process."
+                                f"Failed to reconnect after {MAX_RETRY} attempts, giving up."
                             )
+                            # Mark as closed to prevent further reconnection attempts
+                            self._closed = True
                             if self._handle_disconnected:
                                 self._handle_disconnected(
                                     "Max reconnection attempts exceeded"
                                 )
-                            # Exit process to prevent stuck event loop
-                            import os
-
-                            logger.error(
-                                "Forcing process exit due to unrecoverable connection failure"
-                            )
-                            os._exit(1)
+                            # Note: We intentionally do NOT call os._exit() here.
+                            # Instead, we mark the connection as closed and let the
+                            # application handle the failure through the disconnected
+                            # handler or by checking connection state.
 
                     # Create and track the reconnection task
                     reconnect_task = asyncio.create_task(reconnect_with_retry())

{hypha_rpc-0.20.85 → hypha_rpc-0.20.86}/hypha_rpc.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hypha_rpc
-Version: 0.20.85
+Version: 0.20.86
 Summary: Hypha RPC client for connecting to Hypha server for data management and AI model serving
 Author-email: Wei Ouyang <oeway007@gmail.com>
 Requires-Python: >=3.9

{hypha_rpc-0.20.85 → hypha_rpc-0.20.86}/hypha_rpc.egg-info/SOURCES.txt

@@ -23,6 +23,7 @@ hypha_rpc/utils/schema.py
 hypha_rpc/utils/serve.py
 tests/test_mcp.py
 tests/test_reconnection_runner.py
+tests/test_reconnection_stability.py
 tests/test_schema.py
 tests/test_server_compatibility.py
 tests/test_utils.py

{hypha_rpc-0.20.85 → hypha_rpc-0.20.86}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hypha_rpc"
-version = "0.20.85"
+version = "0.20.86"
 description = "Hypha RPC client for connecting to Hypha server for data management and AI model serving"
 readme = "README.md"
 requires-python = ">=3.9"