xoscar 0.7.16__cp310-cp310-macosx_11_0_arm64.whl → 0.8.0__cp310-cp310-macosx_11_0_arm64.whl

xoscar/backends/communication/socket.py

@@ -31,7 +31,7 @@ from urllib.parse import urlparse
 from ..._utils import to_binary
 from ...constants import XOSCAR_CONNECT_TIMEOUT, XOSCAR_UNIX_SOCKET_DIR
 from ...serialization import AioDeserializer, AioSerializer, deserialize
-from ...utils import classproperty, implements, is_py_312, is_v6_ip
+from ...utils import classproperty, implements, is_py_312, is_py_312_or_above, is_v6_ip
 from .base import Channel, ChannelType, Client, Server
 from .core import register_client, register_server
 from .errors import ChannelClosed
@@ -192,9 +192,9 @@ class _BaseSocketServer(Server, metaclass=ABCMeta):
     @implements(Server.stop)
     async def stop(self):
         self._aio_server.close()
-        # Python 3.12: # https://github.com/python/cpython/issues/104344
-        # `wait_closed` leads to hang
-        if not is_py_312():
+        # Python 3.12+: # https://github.com/python/cpython/issues/104344
+        # `wait_closed` leads to hang in Python 3.12 and 3.13
+        if not is_py_312_or_above():
             await self._aio_server.wait_closed()
         # close all channels
         await asyncio.gather(

xoscar/backends/core.py

@@ -244,7 +244,27 @@ def _cancel_all_tasks(loop):
     for task in to_cancel:
         task.cancel()
 
-    loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True))
+    # In Python 3.13+, we need to use a different approach to avoid deadlocks
+    # when shutting down event loops in threads
+    if hasattr(asyncio, "run"):
+        # For Python 3.13+, use a more robust approach
+        async def _gather_cancelled():
+            await asyncio.gather(*to_cancel, return_exceptions=True)
+
+        try:
+            # Try to run the gather in the current loop context
+            if loop.is_running():
+                # If loop is running, schedule the gather
+                asyncio.run_coroutine_threadsafe(_gather_cancelled(), loop)
+            else:
+                # If loop is not running, we can run it directly
+                loop.run_until_complete(_gather_cancelled())
+        except RuntimeError:
+            # If we can't run the gather, just log and continue
+            logger.debug("Could not gather cancelled tasks during shutdown")
+    else:
+        # For older Python versions, use the original approach
+        loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True))
 
     for task in to_cancel:
         if task.cancelled():
@@ -263,8 +283,15 @@ def _safe_run_forever(loop):
     try:
         loop.run_forever()
     finally:
-        _cancel_all_tasks(loop)
-        loop.stop()
+        try:
+            _cancel_all_tasks(loop)
+        except Exception as e:
+            logger.debug("Error during task cancellation: %s", e)
+        finally:
+            try:
+                loop.stop()
+            except Exception as e:
+                logger.debug("Error stopping loop: %s", e)
 
 
 class ActorCaller:
@@ -273,12 +300,31 @@ class ActorCaller:
     class _RefHolder:
         pass
 
-    _close_loop = asyncio.new_event_loop()
-    _close_thread = threading.Thread(
-        target=_safe_run_forever, args=(_close_loop,), daemon=True
-    )
-    _close_thread.start()
-    atexit.register(_close_loop.call_soon_threadsafe, _close_loop.stop)
+    _close_loop = None
+    _close_thread = None
+    _initialized = False
+
+    @classmethod
+    def _ensure_initialized(cls):
+        if not cls._initialized:
+            cls._close_loop = asyncio.new_event_loop()
+            cls._close_thread = threading.Thread(
+                target=_safe_run_forever, args=(cls._close_loop,), daemon=True
+            )
+            cls._close_thread.start()
+            atexit.register(cls._cleanup)
+            cls._initialized = True
+
+    @classmethod
+    def _cleanup(cls):
+        if cls._close_loop and cls._close_loop.is_running():
+            try:
+                cls._close_loop.call_soon_threadsafe(cls._close_loop.stop)
+                # Give the loop a moment to stop
+                if cls._close_thread:
+                    cls._close_thread.join(timeout=0.5)  # Shorter timeout for tests
+            except Exception as e:
+                logger.debug("Error during cleanup: %s", e)
 
     def __init__(self):
         self._thread_local = threading.local()
@@ -294,6 +340,8 @@ class ActorCaller:
             # If the thread exit, we clean the related actor callers and channels.
 
             def _cleanup():
+                self._ensure_initialized()
+                # Use the background thread for cleanup
                 asyncio.run_coroutine_threadsafe(actor_caller.stop(), self._close_loop)
                 logger.debug(
                     "Clean up the actor caller due to thread exit: %s", thread_info

xoscar/backends/indigen/pool.py

@@ -418,26 +418,70 @@ class MainActorPool(MainActorPoolBase):
     async def kill_sub_pool(
         self, process: asyncio.subprocess.Process, force: bool = False
     ):
+        # First, try to terminate the process gracefully
+        if not force:
+            try:
+                process.terminate()
+                # Wait for graceful termination
+                try:
+                    await asyncio.wait_for(process.wait(), timeout=2.0)
+                except asyncio.TimeoutError:
+                    # Process didn't terminate gracefully, force kill
+                    force = True
+            except ProcessLookupError:
+                # Process already terminated
+                pass
+
+        # Force kill if needed or if graceful termination failed
+        if force:
+            try:
+                process.kill()
+            except ProcessLookupError:
+                # Process already dead
+                pass
+
+        # Ensure process is completely terminated and cleaned up
         try:
-            p = psutil.Process(process.pid)
-        except psutil.NoSuchProcess:
-            return
+            # Wait for process to complete
+            if process.returncode is None:
+                try:
+                    await asyncio.wait_for(process.wait(), timeout=5.0)
+                except asyncio.TimeoutError:
+                    pass
+        except ProcessLookupError:
+            # Process already terminated
+            pass
 
-        if not force:  # pragma: no cover
-            p.terminate()
+        # Python 3.13 specific cleanup for waitpid threads
+        if sys.version_info >= (3, 13):
             try:
-                p.wait(5)
-            except psutil.TimeoutExpired:
+                # Close the transport to clean up waitpid thread
+                if hasattr(process, "_transport") and process._transport:
+                    process._transport.close()
+                # Also try to close the pipe transport if it exists
+                if hasattr(process, "_pipes") and process._pipes:
+                    for pipe in process._pipes.values():
+                        if hasattr(pipe, "close"):
+                            pipe.close()
+            except Exception:
+                # Ignore errors during cleanup
                 pass
 
-        count = 0
-        while p.is_running() and count < 3:
-            count += 1
-            p.kill()
-            if not p.is_running():
-                return
-            logger.info("Sub pool can't be killed: %s", p)
-            time.sleep(0.1)
+        # Additional cleanup using psutil to ensure process tree is terminated
+        try:
+            p = psutil.Process(process.pid)
+            if p.is_running():
+                # Kill the entire process tree
+                for child in p.children(recursive=True):
+                    try:
+                        child.kill()
+                    except psutil.NoSuchProcess:
+                        pass
+                p.kill()
+                p.wait(timeout=2.0)
+        except (psutil.NoSuchProcess, psutil.TimeoutExpired):
+            # Process already dead or couldn't be killed
+            pass
 
     async def is_sub_pool_alive(self, process: asyncio.subprocess.Process):
         return process.returncode is None

xoscar/backends/pool.py

@@ -1337,7 +1337,9 @@ class MainActorPoolBase(ActorPoolBase):
         return pool
 
     async def start_monitor(self):
-        if self._monitor_task is None:
+        # Only start monitor if there are sub processes to monitor
+        # This prevents hanging when n_process=0
+        if self._monitor_task is None and self.sub_processes:
             self._monitor_task = asyncio.create_task(self.monitor_sub_pools())
         return self._monitor_task
 
@@ -1351,7 +1353,12 @@ class MainActorPoolBase(ActorPoolBase):
         self._auto_recover = False
         self._stopped.set()
         if self._monitor_task and not self._monitor_task.done():
-            await self._monitor_task
+            # Cancel the monitor task to ensure it exits immediately
+            self._monitor_task.cancel()
+            try:
+                await self._monitor_task
+            except asyncio.CancelledError:
+                pass  # Expected when cancelling the task
             self._monitor_task = None
         await self.stop_sub_pools()
         await super().stop()
@@ -1406,19 +1413,17 @@ class MainActorPoolBase(ActorPoolBase):
         )
         try:
             if timeout is None:
-                message = await self.call(address, stop_message)
-                if isinstance(message, ErrorMessage):  # pragma: no cover
-                    raise message.as_instanceof_cause()
-            else:
-                call = asyncio.create_task(self.call(address, stop_message))
-                try:
-                    await asyncio.wait_for(call, timeout)
-                except (futures.TimeoutError, asyncio.TimeoutError):  # pragma: no cover
-                    # timeout, just let kill to finish it
-                    force = True
-        except (ConnectionError, ServerClosed):  # pragma: no cover
+                # Use a short timeout for graceful shutdown to avoid hanging
+                timeout = 2.0
+
+            call = asyncio.create_task(self.call(address, stop_message))
+            try:
+                await asyncio.wait_for(call, timeout)
+            except (futures.TimeoutError, asyncio.TimeoutError):
+                force = True
+        except (ConnectionError, ServerClosed):
             # process dead maybe, ignore it
-            pass
+            force = True
         # kill process
         await self.kill_sub_pool(process, force=force)
 

xoscar/backends/test/pool.py

@@ -16,6 +16,7 @@
 from __future__ import annotations
 
 import asyncio
+import sys
 from typing import Any, Optional
 
 from ..communication import DummyServer, gen_local_address
@@ -153,9 +154,44 @@ class TestMainActorPool(MainActorPool):
     async def kill_sub_pool(
         self, process: asyncio.subprocess.Process, force: bool = False
     ):
-        pass
+        # Test pool uses None for processes, so skip if process is None
+        if process is None:
+            return
+
+        if force:
+            try:
+                process.kill()
+            except ProcessLookupError:
+                pass
+
+        # Ensure process is completely terminated and cleaned up
+        try:
+            # Wait for process to complete
+            if process.returncode is None:
+                try:
+                    await asyncio.wait_for(process.wait(), timeout=5.0)
+                except asyncio.TimeoutError:
+                    pass
+        except ProcessLookupError:
+            pass
+
+        # Python 3.13 specific cleanup for waitpid threads
+        if sys.version_info >= (3, 13):
+            try:
+                # Close the transport to clean up waitpid thread
+                if hasattr(process, "_transport") and process._transport:
+                    process._transport.close()
+                # Also try to close the pipe transport if it exists
+                if hasattr(process, "_pipes") and process._pipes:
+                    for pipe in process._pipes.values():
+                        if hasattr(pipe, "close"):
+                            pipe.close()
+            except Exception:
+                # Ignore errors during cleanup
+                pass
 
     async def is_sub_pool_alive(self, process: asyncio.subprocess.Process):
+        # Test pool uses None for processes, so always return True
         return True
 
 

xoscar/collective/backend/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022-2025 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

xoscar/collective/backend/nccl_backend.py

@@ -0,0 +1,160 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# We need to extend cupy's inner class because an actor is a daemonic processes
+# which are not allowed to have children. However, the origin code in cupy
+# will create children processes.
+
+import queue
+import socket
+import threading
+from ctypes import sizeof
+
+from ...utils import lazy_import
+
+cupy = lazy_import("cupy")
+
+if cupy is not None:
+    import cupyx.distributed
+    from cupy.cuda import nccl
+    from cupyx.distributed import _klv_utils, _store, _store_actions
+
+    class ExceptionAwareThreading(threading.Thread):
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            self._exception = None
+            self.q = queue.Queue()
+
+        def run(self):
+            try:
+                super().run()
+                self.q.put(None)
+            except Exception as e:
+                self.q.put(e)
+
+        def join(self):
+            super().join()
+            if not self.q.empty():
+                exception = self.q.get()
+                if exception is not None:
+                    raise exception
+
+    class TCPStore:
+        # This is only used for initialization of nccl so we don't care
+        # too much about performance
+        def __init__(self, world_size):
+            self.storage = {}
+            self._thread = None
+            self._world_size = world_size
+            self._run = 1
+            # For implementing a barrier
+            self._lock = threading.Lock()
+            self._current_barrier = None
+
+        def __del__(self):
+            if not _store._exit_mode:
+                self.stop()
+
+        def _thread_request(self, c_socket):
+            with c_socket:
+                # Receive in KLV format
+                action_bytes = c_socket.recv(sizeof(_klv_utils.action_t))
+                if len(action_bytes) > 0:
+                    action_m = _klv_utils.action_t.from_buffer_copy(action_bytes)
+                    if action_m.length > 256:
+                        raise ValueError("Invalid length for message")
+                    value = bytearray(action_m.value)[: action_m.length]
+                    r = _store_actions.execute_action(action_m.action, value, self)
+                    if r is not None:
+                        c_socket.sendall(r.klv())
+
+        def _server_loop(self, host, port):
+            # This is for minimum info exchange during initialization
+            # a single connection allows to implement locking mechanics easily
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+                s.bind((host, port))
+                s.listen()
+                s.settimeout(0.5)
+                while self._run == 1:
+                    try:
+                        c_socket, addr = s.accept()
+                    except socket.timeout:
+                        continue
+
+                    t = threading.Thread(
+                        target=self._thread_request, args=(c_socket,), daemon=True
+                    )
+                    t.start()
+
+        def run(self, host=_store._DEFAULT_HOST, port=_store._DEFAULT_PORT):
+            # Run the TCP store in a different process
+            t = ExceptionAwareThreading(target=self._server_loop, args=(host, port))
+            t.start()
+            self._thread = t
+
+        def stop(self):
+            if _store._exit_mode:
+                return  # Prevent shutdown errors
+            if self._thread is not None:
+                # acquire the lock
+                self._lock.acquire()
+                self._run = 0
+                self._lock.release()
+                self._thread.join()
+
+    class XoscarNCCLBackend(cupyx.distributed.NCCLBackend):
+        """Interface that uses NVIDIA's NCCL to perform communications.
+
+        Args:
+            n_devices (int): Total number of devices that will be used in the
+                distributed execution.
+            rank (int): Unique id of the GPU that the communicator is associated to
+                its value needs to be `0 <= rank < n_devices`.
+            host (str, optional): host address for the process rendezvous on
+                initialization. Defaults to `"127.0.0.1"`.
+            port (int, optional): port used for the process rendezvous on
+                initialization. Defaults to `13333`.
+            use_mpi(bool, optional): switch between MPI and use the included TCP
+                server for initialization & synchronization. Defaults to `False`.
+        """
+
+        def __init__(
+            self,
+            n_devices,
+            rank,
+            tcpstore,
+            host=_store._DEFAULT_HOST,
+            port=_store._DEFAULT_PORT,
+            use_mpi=False,
+        ):
+            self._tcpstore = tcpstore
+            super().__init__(n_devices, rank, host, port, use_mpi)
+
+        def _init_with_tcp_store(self, n_devices, rank, host, port):
+            nccl_id = None
+            if rank == 0:
+                self._tcpstore.run(host, port)
+                nccl_id = nccl.get_unique_id()
+                # get_unique_id return negative values due to cython issues
+                # with bytes && c strings. We shift them by 128 to
+                # make them positive and send them as bytes to the proxy store
+                shifted_nccl_id = bytes([b + 128 for b in nccl_id])
+                self._store_proxy["nccl_id"] = shifted_nccl_id
+                self._store_proxy.barrier()
+            else:
+                self._store_proxy.barrier()
+                nccl_id = self._store_proxy["nccl_id"]
+                nccl_id = tuple([int(b) - 128 for b in nccl_id])
+            self._comm = nccl.NcclCommunicator(n_devices, nccl_id, rank)

xoscar/utils.py

@@ -470,6 +470,11 @@ def is_py_312():
     return sys.version_info[:2] == (3, 12)
 
 
+@lru_cache
+def is_py_312_or_above():
+    return sys.version_info[:2] >= (3, 12)
+
+
 def is_v4_zero_ip(ip_port_addr: str) -> bool:
     return ip_port_addr.split("://")[-1].startswith("0.0.0.0:")
 

xoscar/virtualenv/platform.py

@@ -39,7 +39,10 @@ def get_cuda_arch() -> Optional[str]:
 
         major, minor = torch.cuda.get_device_capability()
         return f"sm_{major}{minor}"  # e.g. 'sm_80'
-    except (ImportError, AttributeError):
+    except (ImportError, AttributeError, AssertionError):
+        # If no cuda available,
+        # AssertionError("Torch not compiled with CUDA enabled")
+        # will be raised
         return None
 
 

{xoscar-0.7.16.dist-info → xoscar-0.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xoscar
-Version: 0.7.16
+Version: 0.8.0
 Summary: Python actor framework for heterogeneous computing.
 Home-page: http://github.com/xorbitsai/xoscar
 Author: Qin Xuye
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Topic :: Software Development :: Libraries
 Description-Content-Type: text/markdown

{xoscar-0.7.16.dist-info → xoscar-0.8.0.dist-info}/RECORD

@@ -1,7 +1,7 @@
-xoscar-0.7.16.dist-info/RECORD,,
-xoscar-0.7.16.dist-info/WHEEL,sha256=adaOEtyuP97RoRqh2LNimBtjXokZHsp60v8dUwE5uKE,137
-xoscar-0.7.16.dist-info/top_level.txt,sha256=vYlqqY4Nys8Thm1hePIuUv8eQePdULVWMmt7lXtX_ZA,21
-xoscar-0.7.16.dist-info/METADATA,sha256=Yt5bo1VSoHeKy8IPnhVlorXbAkW0Cdws6o4VJZG5Y2E,9135
+xoscar-0.8.0.dist-info/RECORD,,
+xoscar-0.8.0.dist-info/WHEEL,sha256=adaOEtyuP97RoRqh2LNimBtjXokZHsp60v8dUwE5uKE,137
+xoscar-0.8.0.dist-info/top_level.txt,sha256=vYlqqY4Nys8Thm1hePIuUv8eQePdULVWMmt7lXtX_ZA,21
+xoscar-0.8.0.dist-info/METADATA,sha256=XR2fCMVzPBcnQ0vKr6QMLRlwA0NZkKb4xx76-iagFs8,9185
 xoscar/_utils.pyx,sha256=frgVQ5xGp92jBKc4PsPmjOlVsXlKeHWtTOAMfHmBaII,7380
 xoscar/backend.py,sha256=is436OPkZfSpQXaoqTRVta5eoye_pp45RFgCstAk2hU,1850
 xoscar/core.pxd,sha256=I_C2ka7XryyGnnAVXUVm8xfS1gtIrCs6X-9rswgOcUU,1317
@@ -10,18 +10,18 @@ xoscar/context.pxd,sha256=qKa0OyDPZtVymftSh447m-RzFZgmz8rGqQBa7qlauvc,725
 xoscar/batch.py,sha256=DpArS0L3WYJ_HVPG-6hSYEwoAFY1mY2-mlC4Jp5M_Dw,7872
 xoscar/nvutils.py,sha256=qmW4mKLU0WB2yCs198ccQOgLL02zB7Fsa-AotO3NOmg,20412
 xoscar/constants.py,sha256=QHHSREw6uWBBjQDCFqlNfTvBZgniJPGy42KSIsR8Fqw,787
-xoscar/_utils.cpython-310-darwin.so,sha256=G_cpB6KGlOnZGqCldCvZ8AWdvO4tOgqllZYNqXOwGnE,169584
+xoscar/_utils.cpython-310-darwin.so,sha256=xC42ijCc1Pi1H0JOrQJkFgEMJs-amW82BGcA2IUNmzQ,169584
 xoscar/__init__.py,sha256=sy7Wtn2EuQZI0I4Az_MfsBVZm4G0DRj46qRyExgmnJk,1622
 xoscar/api.py,sha256=zxNqOjGiTIKuAip9WJ0LOoM7yevD6P5rb-sLynpZ2Zo,14648
-xoscar/utils.py,sha256=MaKiW4Vphwhh8c0yoqN8G8hbJr1zXgpf49EdvmGc1ZU,16500
+xoscar/utils.py,sha256=vo1DS6xHnvZ9jDS9qpGFFN4GCJDo6nyhMINWOna5dp8,16582
 xoscar/debug.py,sha256=9Z8SgE2WaKYQcyDo-5-DxEJQ533v7kWjrvCd28pSx3E,5069
 xoscar/libcpp.pxd,sha256=DJqBxLFOKL4iRr9Kale5UH3rbvPRD1x5bTSOPHFpz9I,1147
 xoscar/context.pyx,sha256=8CdgPnWcE9eOp3N600WgDQ03MCi8P73eUOGcfV7Zksg,10942
 xoscar/errors.py,sha256=wBlQOKsXf0Fc4skN39tDie0YZT-VIAuLNRgoDl2pZcA,1241
 xoscar/core.pyx,sha256=phN-yYV0A0QI8WFi2jCu0nc4CnShTepfDi0V7ZrLYPY,22092
 xoscar/driver.py,sha256=498fowtJr6b3FE8FIOA_Tc1Vwx88nfZw7p0FxrML0h4,1372
-xoscar/context.cpython-310-darwin.so,sha256=wKf90z7U7qgSucVXNH-FDQfrCDTa4SufxFzjoBt2WpM,211328
-xoscar/core.cpython-310-darwin.so,sha256=NkyiXj_qyWjKiTsF8In-8FkrQXgtw5tefCCj3IaS58E,409608
+xoscar/context.cpython-310-darwin.so,sha256=Aa5Xe4alRLdaFTlhCZaGFv8YSmbj89StkO_BRUmuUPk,211328
+xoscar/core.cpython-310-darwin.so,sha256=ZSPlzx0miEEuVKR5RMeM3ESkCAjFMbwxSw1gjra02p0,409608
 xoscar/profiling.py,sha256=BC5OF0HzSaXv8V7w-y-B8r5gV5DgxHFoTEIF6jCMioQ,8015
 xoscar/_utils.pxd,sha256=5KYAL3jfPdejsHnrGGT2s--ZUX5SXznQWpHVSno429k,1157
 xoscar/metrics/__init__.py,sha256=9Badi7rxYikGm2dQiNCrj9GgMRBxwuR3JaEKcFZmfak,705
@@ -32,12 +32,14 @@ xoscar/metrics/backends/prometheus/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4
 xoscar/metrics/backends/prometheus/prometheus_metric.py,sha256=MxoMvVrg0pOkKpkjJ0PcAuEaaEJR2FZljmPrLjQ1-oc,2050
 xoscar/metrics/backends/console/console_metric.py,sha256=y5CCtH33j3AqI5_Uhwi4mgOcAhyhb4cWv_YvR6fxcbQ,2082
 xoscar/metrics/backends/console/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xoscar/collective/xoscar_pygloo.cpython-310-darwin.so,sha256=aQTSv8R-g0HtppHsnjfGJRLw87dWVMPV_urHEECI240,1144288
+xoscar/collective/xoscar_pygloo.cpython-310-darwin.so,sha256=pDCrh35TJYP1qlzbq1GRUayCSxh-jxWlWdSjtYSuEzE,1127552
 xoscar/collective/__init__.py,sha256=XsClIkO_3Jd8GDifYuAbZCmJLAo9ZqGvnjUn9iuogmU,774
 xoscar/collective/core.py,sha256=NVR-7Iaq3aDPCN6fgXcq9Ew6uFEszRwxYqmUG9FLcws,23502
 xoscar/collective/common.py,sha256=INAnISbfnRicbbbDHTqbSr9ITb89ZphH5BUkSpEdXXU,3561
 xoscar/collective/utils.py,sha256=3S4qF4JEnAUD3RiWVBUj-ZptL83CBSwGYyVZyIasAsE,1178
 xoscar/collective/process_group.py,sha256=zy7LcIFnEcmrcxuECI89v0bQlUbSqQMkVyBw468WBnk,22599
+xoscar/collective/backend/nccl_backend.py,sha256=7VvjAVTkr6qWJC1CztzJ5CN9USGJkstO-RAbaPKQA-Y,6280
+xoscar/collective/backend/__init__.py,sha256=CyLLkbImZouAk4lePIgKXT4WQoqyauIEwdqea5IOUVU,581
 xoscar/serialization/exception.py,sha256=Jy8Lsk0z-VJyEUaWeuZIwkmxqaoB-nLKMa1D15Cl4js,1634
 xoscar/serialization/pyfury.py,sha256=sifOnVMYoS82PzZEkzkfxesmMHei23k5UAUUKUyoOYQ,1163
 xoscar/serialization/core.pxd,sha256=k4RoJgX5E5LGs4jdCQ7vvcn26MabXbrWoWhkO49X6YI,985
@@ -47,33 +49,33 @@ xoscar/serialization/cuda.py,sha256=iFUEnN4SiquBIhyieyOrfw3TnKnW-tU_vYgqOxO_DrA,
 xoscar/serialization/scipy.py,sha256=yOEi0NB8cqQ6e2UnCZ1w006RsB7T725tIL-DM_hNcsU,2482
 xoscar/serialization/aio.py,sha256=5DySPgDxU43ec7_5Ct44-Oqt7YNSJBfuf8VdQgQlChA,4731
 xoscar/serialization/core.pyx,sha256=bjR-zXGm9qersk7kYPzpjpMIxDl_Auur4BCubRfKmfA,29626
-xoscar/serialization/core.cpython-310-darwin.so,sha256=IW1DQy4ULMGRQtqvzlBVVPWKxo2mFOjuwH5AVFgK7fI,363288
+xoscar/serialization/core.cpython-310-darwin.so,sha256=G1JW-fCbnCIAzGIlx6PJqTC7kdRBYnat2iXC1rON9VU,363288
 xoscar/serialization/mlx.py,sha256=tRu_7o6RizdRhbr88EasHrZtShimAsLy3pIEO-by29o,2118
 xoscar/backends/config.py,sha256=4tZMiXAMMS8qQ4SX_LjONLtSQVfZTx3m-IK3EqbkYdk,5375
 xoscar/backends/allocate_strategy.py,sha256=tC1Nbq2tJohahUwd-zoRYHEDX65wyuX8tmeY45uWj_w,4845
-xoscar/backends/message.cpython-310-darwin.so,sha256=16zUFWTFKKRcVfsT8a7ioeWR6hIIeBb1JnqjEE98MZM,349168
+xoscar/backends/message.cpython-310-darwin.so,sha256=Sr1-FSsEGCIcum9TUAQzU3g1wuqWKb3l4xT66iImPGI,349424
 xoscar/backends/__init__.py,sha256=VHEBQcUWM5bj027W8EUf9PiJUAP7JoMrRw3Tsvy5ySw,643
-xoscar/backends/core.py,sha256=EH-fHlV9x3bnruEHaUtGYO7osKLfLJ4AQHtuzA_mr2g,10857
+xoscar/backends/core.py,sha256=hhowZJ1PmsmK1ap3IkIny-50cM3qawYbtUHVSBxzfPc,12845
 xoscar/backends/context.py,sha256=XfDPG2eDhAhE6hWBEkEsHTnyyOYN9R3houlMjAL7BFw,16329
 xoscar/backends/router.py,sha256=MVl5naz-FYf-Wla7XRn3kRxOpWV0SjKDsKNluifVA8M,10532
 xoscar/backends/message.pyx,sha256=krGVtZ1YDaZX8yWhaNHwZiudQooLvcGlw6x3Sq7jxjE,19685
-xoscar/backends/pool.py,sha256=nrh8qobaukkjUOOOTR9t90i-wbXlgma3TNRjvwkwmcg,60528
+xoscar/backends/pool.py,sha256=7yyU5EzU0Izf3j8ifGovpGbGR3KrCoYIU0L3JH_e9WM,60683
 xoscar/backends/indigen/backend.py,sha256=znl_fZzWGEtLH8hZ9j9Kkf0fva25jEem2_KO7I1RVvc,1612
 xoscar/backends/indigen/shared_memory.py,sha256=wqbckbgnd0qNm5KzlP_hklF3F_n8fKnCehSox5uMwNs,19082
 xoscar/backends/indigen/__init__.py,sha256=tKHP5ClzedBRBpZsLRVErR3EUNbbDm4CY4u0rCFJr44,685
 xoscar/backends/indigen/fate_sharing.py,sha256=3QUHwq5Cjk9oCKFUISvkqHaoxWZIaXcq8JNOetdBl-A,8655
 xoscar/backends/indigen/driver.py,sha256=VGzkacYKykegW5qhCuhx01gdgBZEKJjNIyfNCnA6Nm8,952
-xoscar/backends/indigen/pool.py,sha256=t8zQ1i6Kf3JEy8EMMX7vTbKakilkn4TuxPdsxaxh27A,16573
+xoscar/backends/indigen/pool.py,sha256=RAbXNkb8y5QElAY2zcD6NBb8XBT5bt2GqFc12H3SwqY,18504
 xoscar/backends/indigen/__main__.py,sha256=-pfio-Y4Ogbk6lBFksH-gRatp-N6sZ7wuNc-i2YsLJc,510
 xoscar/backends/test/backend.py,sha256=nv9WFhH5Bbq4Q1HB9yfpciZBaeHT4IQAtzugBWESrUY,1263
 xoscar/backends/test/__init__.py,sha256=j2ZfD6prD9WjUxRUDC7Eq5Z7N7TkL6fFr59oNyc_vY4,682
-xoscar/backends/test/pool.py,sha256=1MrdNnQZkeBJMYrn3NGUe1VpvsUM_HlTm3l7SSynzpo,6882
+xoscar/backends/test/pool.py,sha256=tPq-MKHmiB9SOVo64y6Nv-U3evdxVQZuuWLYR-fj3ZA,8257
 xoscar/backends/communication/ucx.py,sha256=_Dp9Ld2MWIa1txSGMnmfYwJDT0esxS-GOd2FQ4BdHiM,19960
 xoscar/backends/communication/__init__.py,sha256=oFIg83Ga93-AhrG52TE85Z2LgpGZu1RCgQu1RWi62zQ,1063
 xoscar/backends/communication/core.py,sha256=sJeE3foRIqVPXldzYpFKHDSsabfAIFBU4JuXY4OyklY,2130
 xoscar/backends/communication/utils.py,sha256=AmovE-hmWLXNCPwHafYuaRjOk8m42BUyT3XBqfXQRVI,3664
 xoscar/backends/communication/errors.py,sha256=V3CdBe2xX9Rwv32f2dH2Msc84yaUhlyerZ42-739o1Q,723
-xoscar/backends/communication/socket.py,sha256=6Pf9RJWHuvobjZ1eAU6bUuGY7PzYga3Vyc4PAKF-k2M,14428
+xoscar/backends/communication/socket.py,sha256=8GW-7HNnZE7tqvLyU6L9B75kaQvI4atc6AGwVHq1IGE,14482
 xoscar/backends/communication/dummy.py,sha256=6kLkxjNk4xTQ-IlNZD6cftNCx5UsGOur2jk7ikrNUCg,8157
 xoscar/backends/communication/base.py,sha256=0P4Tr35GSWpRp394e9jVWUUoKKa-gIk177eYPw1BnSU,7421
 xoscar/aio/__init__.py,sha256=kViDKR_kJe59VQViHITKEfBcIgN4ZJblUyd8zl0E3ZI,675
@@ -83,6 +85,6 @@ xoscar/aio/parallelism.py,sha256=VSsjk8wP-Bw7tLeUsTyLVNgp91thjxEfE3pCrw_vF5Q,129
 xoscar/aio/base.py,sha256=9j0f1piwfE5R5GIvV212vSD03ixdaeSzSSsO2kxJZVE,2249
 xoscar/virtualenv/__init__.py,sha256=65t9_X1DvbanNjFy366SiiWZrRTpa9SXWMXPmqayE-4,1117
 xoscar/virtualenv/core.py,sha256=Ij36UQaej9fFaz1PfqkEtL1ss8yBribXHcWT115kH-o,8098
-xoscar/virtualenv/platform.py,sha256=3f5EQEXbq_sf4pRJ_0lg4y1V_22qadUdQjmVjIaZaoU,1403
+xoscar/virtualenv/platform.py,sha256=JGEp6tbCeACr3MC9UAdaOGJIJEKjWFRtScpKuGX0_Hs,1541
 xoscar/virtualenv/utils.py,sha256=qKHw7Gg0n3JuzKFjhBnftPq2QWlgNJLk1sGPr5GzamM,2875
 xoscar/virtualenv/uv.py,sha256=VBw045LN8gYMLgjaazt7-tnwBveWr7YYE2zjDsL18h0,11091