torchmonarch-nightly 2025.6.6__cp310-cp310-manylinux2014_x86_64.whl → 2025.6.8__cp310-cp310-manylinux2014_x86_64.whl

monarch/actor_mesh.py

@@ -39,7 +39,7 @@ from typing import (
 import monarch
 from monarch import ActorFuture as Future
 
-from monarch._rust_bindings.monarch_hyperactor.actor import PythonMessage
+from monarch._rust_bindings.monarch_hyperactor.actor import PanicFlag, PythonMessage
 from monarch._rust_bindings.monarch_hyperactor.actor_mesh import PythonActorMesh
 from monarch._rust_bindings.monarch_hyperactor.mailbox import (
     Mailbox,
@@ -264,9 +264,9 @@ class Endpoint(Generic[P, R]):
         return self.choose(*args, **kwargs)
 
     def call(self, *args: P.args, **kwargs: P.kwargs) -> "Future[ValueMesh[R]]":
-        p, r = port(self, kind=RankedPort)
+        p, r = port(self)
         # pyre-ignore
-        send(self, args, kwargs, port=p)
+        send(self, args, kwargs, port=p, rank_in_response=True)
 
         async def process():
             results = [None] * len(self._actor_mesh)
@@ -361,8 +361,9 @@ def send(
     endpoint: Endpoint[P, R],
     args: Tuple[Any, ...],
     kwargs: Dict[str, Any],
-    port: "Optional[Port]" = None,
+    port: "Optional[PortId]" = None,
     selection: Selection = "all",
+    rank_in_response: bool = False,
 ) -> None:
     """
     Fire-and-forget broadcast invocation of the endpoint across all actors in the mesh.
@@ -370,7 +371,9 @@ def send(
     This sends the message to all actors but does not wait for any result.
     """
     endpoint._signature.bind(None, *args, **kwargs)
-    message = PythonMessage(endpoint._name, _pickle((args, kwargs, port)))
+    message = PythonMessage(
+        endpoint._name, _pickle((args, kwargs)), port, rank_in_response
+    )
     endpoint._actor_mesh.cast(message, selection)
 
 
@@ -392,28 +395,29 @@ def endpoint(
 
 
 class Port:
-    def __init__(self, port: PortId, mailbox: Mailbox) -> None:
+    def __init__(self, port: PortId, mailbox: Mailbox, rank_in_response: bool) -> None:
         self._port = port
         self._mailbox = mailbox
+        self._rank_in_response = rank_in_response
 
     def send(self, method: str, obj: object) -> None:
+        if self._rank_in_response:
+            obj = (MonarchContext.get().point.rank, obj)
         self._mailbox.post(
             self._port,
-            PythonMessage(method, _pickle(obj)),
+            PythonMessage(method, _pickle(obj), None),
         )
 
 
 # advance lower-level API for sending messages. This is intentially
 # not part of the Endpoint API because they way it accepts arguments
 # and handles concerns is different.
-def port(
-    endpoint: Endpoint[P, R], once=False, kind=Port
-) -> Tuple["Port", "PortReceiver[R]"]:
+def port(endpoint: Endpoint[P, R], once=False) -> Tuple["PortId", "PortReceiver[R]"]:
     handle, receiver = (
         endpoint._mailbox.open_once_port() if once else endpoint._mailbox.open_port()
     )
     port_id: PortId = handle.bind()
-    return kind(port_id, endpoint._mailbox), PortReceiver(endpoint._mailbox, receiver)
+    return port_id, PortReceiver(endpoint._mailbox, receiver)
 
 
 class PortReceiver(Generic[R]):
@@ -439,8 +443,8 @@ class PortReceiver(Generic[R]):
         else:
             assert msg.method == "exception"
             if isinstance(payload, tuple):
-                # If we're receiving on a RankedPort, raise the exception and ignore the rank.
-                # pyre-ignore do something more structured here
+                # If the payload is a tuple, it's because we requested the rank
+                # to be included in the response; just ignore it.
                 raise payload[1]
             else:
                 # pyre-ignore
@@ -453,21 +457,16 @@ class PortReceiver(Generic[R]):
 singleton_shape = Shape([], NDSlice(offset=0, sizes=[], strides=[]))
 
 
-class RankedPort(Port):
-    def send(self, method: str, obj: object) -> None:
-        super().send(method, (MonarchContext.get().point.rank, obj))
-
-
 class _Actor:
     def __init__(self) -> None:
         self.instance: object | None = None
         self.active_requests: asyncio.Queue[asyncio.Future[object]] = asyncio.Queue()
-        self.complete_task: object | None = None
+        self.complete_task: asyncio.Task | None = None
 
     def handle(
-        self, mailbox: Mailbox, message: PythonMessage
+        self, mailbox: Mailbox, message: PythonMessage, panic_flag: PanicFlag
     ) -> Optional[Coroutine[Any, Any, Any]]:
-        return self.handle_cast(mailbox, 0, singleton_shape, message)
+        return self.handle_cast(mailbox, 0, singleton_shape, message, panic_flag)
 
     def handle_cast(
         self,
@@ -475,14 +474,18 @@ class _Actor:
         rank: int,
         shape: Shape,
         message: PythonMessage,
+        panic_flag: PanicFlag,
     ) -> Optional[Coroutine[Any, Any, Any]]:
-        port = None
+        port = (
+            Port(message.response_port, mailbox, message.rank_in_response)
+            if message.response_port
+            else None
+        )
         try:
-            args, kwargs, port = _unpickle(message.message, mailbox)
-
             ctx = MonarchContext(mailbox, mailbox.actor_id.proc_id, Point(rank, shape))
             _context.set(ctx)
 
+            args, kwargs = _unpickle(message.message, mailbox)
             if message.method == "__init__":
                 Class, *args = args
                 self.instance = Class(*args, **kwargs)
@@ -495,10 +498,10 @@ class _Actor:
                         port.send("result", result)
                     return None
 
-                return self.run_async(ctx, self.run_task(port, result))
+                return self.run_async(ctx, self.run_task(port, result, panic_flag))
         except Exception as e:
             traceback.print_exc()
-            s = ActorMeshRefCallFailedException(e)
+            s = ActorError(e)
 
             # The exception is delivered to exactly one of:
             # (1) our caller, (2) our supervisor
@@ -510,17 +513,17 @@ class _Actor:
     async def run_async(self, ctx, coroutine):
         _context.set(ctx)
         if self.complete_task is None:
-            asyncio.create_task(self._complete())
+            self.complete_task = asyncio.create_task(self._complete())
         await self.active_requests.put(create_eager_task(coroutine))
 
-    async def run_task(self, port, coroutine):
+    async def run_task(self, port, coroutine, panic_flag):
         try:
             result = await coroutine
             if port is not None:
                 port.send("result", result)
         except Exception as e:
             traceback.print_exc()
-            s = ActorMeshRefCallFailedException(e)
+            s = ActorError(e)
 
             # The exception is delivered to exactly one of:
             # (1) our caller, (2) our supervisor
@@ -528,6 +531,16 @@ class _Actor:
                 port.send("exception", s)
             else:
                 raise s from None
+        except BaseException as e:
+            # A BaseException can be thrown in the case of a Rust panic.
+            # In this case, we need a way to signal the panic to the Rust side.
+            # See [Panics in async endpoints]
+            try:
+                panic_flag.signal_panic(e)
+            except Exception:
+                # The channel might be closed if the Rust side has already detected the error
+                pass
+            raise
 
     async def _complete(self) -> None:
         while True:
@@ -627,7 +640,6 @@ class ActorMeshRef(MeshTrait):
             null_func,
             self._mailbox,
         )
-        # pyre-ignore
         send(ep, (self._class, *args), kwargs)
 
     def __reduce_ex__(
@@ -655,7 +667,7 @@ class ActorMeshRef(MeshTrait):
         )
 
 
-class ActorMeshRefCallFailedException(Exception):
+class ActorError(Exception):
     """
     Deterministic problem with the user's code.
     For example, an OOM resulting in trying to allocate too much GPU memory, or violating

monarch/bootstrap_main.py

@@ -53,6 +53,9 @@ def invoke_main():
                     record.levelno,
                 )
 
+    if os.environ.get("MONARCH_ERROR_DURING_BOOTSTRAP_FOR_TESTING") == "1":
+        raise RuntimeError("Error during bootstrap for testing")
+
     # forward logs to rust tracing. Defaults to on.
     if os.environ.get("MONARCH_PYTHON_LOG_TRACING", "1") == "1":
         logging.root.addHandler(TracingForwarder())

monarch/common/device_mesh.py

@@ -97,7 +97,7 @@ class RemoteProcessGroup(Referenceable):
         self._drop_ref()
 
     def size(self):
-        return self.device_mesh.numdevices(self.dims)
+        return self.device_mesh.size(self.dims)
 
     def _drop_ref(self):
         if self.ref is None:
@@ -172,21 +172,6 @@ class DeviceMesh(Referenceable, MeshTrait):
         self.ref = None
         self._active_mesh_context = None
 
-    def numdevices(self, dims: Optional[Dims] = None) -> int:
-        """
-        Returns the number of devices (total) of the subset of mesh asked for.
-        If dims is None, returns the total number of devices in the mesh.
-        """
-        if dims is None:
-            dims = self.names
-        missing_dims = set(dims) - set(self.names)
-        if missing_dims:
-            raise ValueError(f"Dimensions not found: {', '.join(missing_dims)}")
-        product = 1
-        for dim in dims:
-            product *= self.size(dim)
-        return product
-
     def define_remotely(self):
         if self.ref is None:
             self.ref = self.client.new_ref()
@@ -275,25 +260,10 @@ class DeviceMesh(Referenceable, MeshTrait):
             combined_rank += self.rank(dim)
         return combined_rank
 
-    def size(self, dim: Union[str, Sequence[str]]) -> int:
-        if isinstance(dim, str):
-            if dim not in self.names:
-                raise KeyError(f"{self} does not have dimension {repr(dim)}")
-            return self.processes.sizes[self.names.index(dim)]
-        else:
-            p = 1
-            for d in dim:
-                p *= self.size(d)
-            return p
-
     @property
     def ranks(self) -> dict[str, int]:
         return {dim: self.rank(dim) for dim in self.names}
 
-    @property
-    def sizes(self) -> dict[str, int]:
-        return dict(zip(self.names, self.processes.sizes))
-
     def process_idx(self):
         self.define_remotely()
         return _remote(

monarch/common/shape.py

@@ -8,7 +8,7 @@ import itertools
 import operator
 from abc import ABC, abstractmethod
 
-from typing import Dict, Generator, Sequence, Tuple
+from typing import Dict, Generator, Sequence, Tuple, Union
 
 from monarch._rust_bindings.monarch_hyperactor.shape import Shape, Slice
 
@@ -200,5 +200,27 @@ class MeshTrait(ABC):
         """
         return self.split(**{k: (v,) for k, v in kwargs.items()})
 
+    def size(self, dim: Union[None, str, Sequence[str]] = None) -> int:
+        """
+        Returns the number of elements (total) of the subset of mesh asked for.
+        If dims is None, returns the total number of devices in the mesh.
+        """
+
+        if dim is None:
+            dim = self._labels
+        if isinstance(dim, str):
+            if dim not in self._labels:
+                raise KeyError(f"{self} does not have dimension {repr(dim)}")
+            return self._ndslice.sizes[self._labels.index(dim)]
+        else:
+            p = 1
+            for d in dim:
+                p *= self.size(d)
+            return p
+
+    @property
+    def sizes(self) -> dict[str, int]:
+        return dict(zip(self._labels, self._ndslice.sizes))
+
 
 __all__ = ["NDSlice", "Shape", "MeshTrait"]

monarch/proc_mesh.py

@@ -18,9 +18,11 @@ from monarch._rust_bindings.hyperactor_extension.alloc import (  # @manual=//mon
 )
 from monarch._rust_bindings.monarch_hyperactor.mailbox import Mailbox
 from monarch._rust_bindings.monarch_hyperactor.proc_mesh import ProcMesh as HyProcMesh
+from monarch._rust_bindings.monarch_hyperactor.shape import Shape
 from monarch.actor_mesh import _Actor, _ActorMeshRefImpl, Actor, ActorMeshRef
 
 from monarch.common._device_utils import _local_device_count
+from monarch.common.shape import MeshTrait
 from monarch.rdma import RDMAManager
 
 T = TypeVar("T")
@@ -40,12 +42,23 @@ def _allocate_blocking(alloc: Alloc) -> "ProcMesh":
     return ProcMesh(HyProcMesh.allocate_blocking(alloc))
 
 
-class ProcMesh:
+class ProcMesh(MeshTrait):
     def __init__(self, hy_proc_mesh: HyProcMesh) -> None:
         self._proc_mesh = hy_proc_mesh
         self._mailbox: Mailbox = self._proc_mesh.client
         self._rdma_manager = self._spawn_blocking("rdma_manager", RDMAManager)
 
+    @property
+    def _ndslice(self):
+        return self._proc_mesh.shape.ndslice
+
+    @property
+    def _labels(self):
+        return self._proc_mesh.shape.labels
+
+    def _new_with_shape(self, shape: Shape) -> "ProcMesh":
+        raise NotImplementedError("ProcMesh slicing is not implemeted yet.")
+
     def spawn(self, name: str, Class: Type[T], *args: Any, **kwargs: Any) -> Future[T]:
         return Future(
             lambda: self._spawn_nonblocking(name, Class, *args, **kwargs),

monarch/rdma.py

@@ -6,10 +6,7 @@
 
 import ctypes
 
-import traceback
-
 from dataclasses import dataclass
-from traceback import extract_tb, StackSummary
 from typing import cast, Dict, Optional, Tuple
 
 import torch
@@ -163,28 +160,3 @@ class RDMABuffer:
             src.numel(),
         )
         await RDMAManager.on_proc(self.proc_id).put.call_one(self.addr, offset, bytes)
-
-
-class ActorMeshRefCallFailedException(Exception):
-    """
-    Deterministic problem with the user's code.
-    For example, an OOM resulting in trying to allocate too much GPU memory, or violating
-    some invariant enforced by the various APIs.
-    """
-
-    def __init__(
-        self,
-        exception: Exception,
-        message: str = "A remote service call has failed asynchronously.",
-    ) -> None:
-        self.exception = exception
-        self.actor_mesh_ref_frames: StackSummary = extract_tb(exception.__traceback__)
-        self.message = message
-
-    def __str__(self) -> str:
-        exe = str(self.exception)
-        actor_mesh_ref_tb = "".join(traceback.format_list(self.actor_mesh_ref_frames))
-        return (
-            f"{self.message}\n"
-            f"Traceback of where the service call failed (most recent call last):\n{actor_mesh_ref_tb}{type(self.exception).__name__}: {exe}"
-        )

tests/error_test_binary.py

@@ -7,6 +7,8 @@
 import ctypes
 import sys
 
+import click
+
 from monarch._rust_bindings.monarch_extension.panic import panicking_function
 
 from monarch.actor_mesh import Actor, endpoint
@@ -115,24 +117,33 @@ def _run_error_test(num_procs, sync_endpoint, endpoint_name):
     asyncio.run(run_test())
 
 
+@click.group()
 def main():
-    import argparse
+    pass
 
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--num-procs", type=int)
-    parser.add_argument("--sync-test-impl", type=bool)
-    parser.add_argument("--sync-endpoint", type=bool)
-    parser.add_argument("--endpoint-name", type=str)
-    args = parser.parse_args()
 
+@main.command("error-endpoint")
+@click.option("--num-procs", type=int, required=True)
+@click.option("--sync-test-impl", type=bool, required=True)
+@click.option("--sync-endpoint", type=bool, required=True)
+@click.option("--endpoint-name", type=str, required=True)
+def error_endpoint(num_procs, sync_test_impl, sync_endpoint, endpoint_name):
     print(
-        f"Running segfault test: {args.num_procs=} {args.sync_test_impl=} {args.sync_endpoint=}, {args.endpoint_name=}"
+        f"Running segfault test: {num_procs=} {sync_test_impl=} {sync_endpoint=}, {endpoint_name=}"
     )
 
-    if args.sync_test_impl:
-        _run_error_test_sync(args.num_procs, args.sync_endpoint, args.endpoint_name)
+    if sync_test_impl:
+        _run_error_test_sync(num_procs, sync_endpoint, endpoint_name)
     else:
-        _run_error_test(args.num_procs, args.sync_endpoint, args.endpoint_name)
+        _run_error_test(num_procs, sync_endpoint, endpoint_name)
+
+
+@main.command("error-bootstrap")
+def error_bootstrap():
+    print("I actually ran")
+    sys.stdout.flush()
+
+    proc_mesh(gpus=4, env={"MONARCH_ERROR_DURING_BOOTSTRAP_FOR_TESTING": "1"}).get()
 
 
 if __name__ == "__main__":

tests/test_actor_error.py

@@ -8,47 +8,76 @@ import importlib.resources
 import subprocess
 
 import pytest
-from monarch.actor_mesh import Actor, ActorMeshRefCallFailedException, endpoint
+from monarch.actor_mesh import Actor, ActorError, endpoint
 
 from monarch.proc_mesh import proc_mesh
 
 
 class ExceptionActor(Actor):
-    """An actor that has endpoints which raise exceptions."""
-
     @endpoint
     async def raise_exception(self) -> None:
-        """Endpoint that raises an exception."""
         raise Exception("This is a test exception")
 
+    @endpoint
+    async def print_value(self, value) -> None:
+        """Endpoint that takes a value and prints it."""
+        print(f"Value received: {value}")
+        return value
 
-class ExceptionActorSync(Actor):
-    """An actor that has endpoints which raise exceptions."""
 
+class ExceptionActorSync(Actor):
     @endpoint  # pyre-ignore
     def raise_exception(self) -> None:
-        """Endpoint that raises an exception."""
         raise Exception("This is a test exception")
 
 
+class BrokenPickleClass:
+    """A class that can be configured to raise exceptions during pickling/unpickling."""
+
+    def __init__(
+        self,
+        raise_on_getstate=False,
+        raise_on_setstate=False,
+        exception_message="Pickle error",
+    ):
+        self.raise_on_getstate = raise_on_getstate
+        self.raise_on_setstate = raise_on_setstate
+        self.exception_message = exception_message
+        self.value = "test_value"
+
+    def __getstate__(self):
+        """Called when pickling the object."""
+        if self.raise_on_getstate:
+            raise RuntimeError(f"__getstate__ error: {self.exception_message}")
+        return {
+            "raise_on_getstate": self.raise_on_getstate,
+            "raise_on_setstate": self.raise_on_setstate,
+            "exception_message": self.exception_message,
+            "value": self.value,
+        }
+
+    def __setstate__(self, state):
+        """Called when unpickling the object."""
+        if state.get("raise_on_setstate", False):
+            raise RuntimeError(
+                f"__setstate__ error: {state.get('exception_message', 'Unpickle error')}"
+            )
+        self.__dict__.update(state)
+
+
 @pytest.mark.parametrize(
-    "actor_class,actor_name",
-    [
-        (ExceptionActor, "exception_actor_async_call"),
-        (ExceptionActorSync, "exception_actor_sync_call"),
-    ],
+    "actor_class",
+    [ExceptionActor, ExceptionActorSync],
 )
 @pytest.mark.parametrize("num_procs", [1, 2])
-async def test_actor_exception(actor_class, actor_name, num_procs):
+async def test_actor_exception(actor_class, num_procs):
     """
     Test that exceptions raised in actor endpoints are propagated to the client.
     """
     proc = await proc_mesh(gpus=num_procs)
-    exception_actor = await proc.spawn(actor_name, actor_class)
+    exception_actor = await proc.spawn("exception_actor", actor_class)
 
-    with pytest.raises(
-        ActorMeshRefCallFailedException, match="This is a test exception"
-    ):
+    with pytest.raises(ActorError, match="This is a test exception"):
         if num_procs == 1:
             await exception_actor.raise_exception.call_one()
         else:
@@ -56,23 +85,18 @@ async def test_actor_exception(actor_class, actor_name, num_procs):
 
 
 @pytest.mark.parametrize(
-    "actor_class,actor_name",
-    [
-        (ExceptionActor, "exception_actor_async_call"),
-        (ExceptionActorSync, "exception_actor_sync_call"),
-    ],
+    "actor_class",
+    [ExceptionActor, ExceptionActorSync],
 )
 @pytest.mark.parametrize("num_procs", [1, 2])
-def test_actor_exception_sync(actor_class, actor_name, num_procs):
+def test_actor_exception_sync(actor_class, num_procs):
     """
     Test that exceptions raised in actor endpoints are propagated to the client.
     """
     proc = proc_mesh(gpus=num_procs).get()
-    exception_actor = proc.spawn(actor_name, actor_class).get()
+    exception_actor = proc.spawn("exception_actor", actor_class).get()
 
-    with pytest.raises(
-        ActorMeshRefCallFailedException, match="This is a test exception"
-    ):
+    with pytest.raises(ActorError, match="This is a test exception"):
         if num_procs == 1:
             exception_actor.raise_exception.call_one().get()
         else:
@@ -85,28 +109,102 @@ def test_actor_exception_sync(actor_class, actor_name, num_procs):
 @pytest.mark.parametrize("sync_endpoint", [False, True])
 @pytest.mark.parametrize("sync_test_impl", [False, True])
 @pytest.mark.parametrize("endpoint_name", ["cause_segfault", "cause_panic"])
-def test_actor_segfault(num_procs, sync_endpoint, sync_test_impl, endpoint_name):
+def test_actor_supervision(num_procs, sync_endpoint, sync_test_impl, endpoint_name):
     """
-    Test that segfaults in actor endpoints result in a non-zero exit code.
-    This test spawns a subprocess that will segfault and checks its exit code.
+    Test that an endpoint causing spontaenous process exit is handled by the supervisor.
 
-    Tests both ExceptionActor and ExceptionActorSync using async API.
+    Today, these events are delivered to the client and cause the client process
+    to exit with a non-zero code, so the only way we can test it is via a
+    subprocess harness.
     """
     # Run the segfault test in a subprocess
     test_bin = importlib.resources.files("monarch.python.tests").joinpath("test_bin")
     cmd = [
         str(test_bin),
+        "error-endpoint",
         f"--num-procs={num_procs}",
         f"--sync-endpoint={sync_endpoint}",
         f"--sync-test-impl={sync_test_impl}",
         f"--endpoint-name={endpoint_name}",
     ]
-    process = subprocess.run(cmd, capture_output=True, timeout=60)
-    print(process.stdout.decode())
-    print(process.stderr.decode())
+    try:
+        process = subprocess.run(cmd, capture_output=True, timeout=180)
+    except subprocess.TimeoutExpired as e:
+        print("timeout expired")
+        if e.stdout is not None:
+            print(e.stdout.decode())
+        if e.stderr is not None:
+            print(e.stderr.decode())
+        raise
+
+    # Assert that the subprocess exited with a non-zero code
+    assert "I actually ran" in process.stdout.decode()
+    assert (
+        process.returncode != 0
+    ), f"Expected non-zero exit code, got {process.returncode}"
+
+
+# oss_skip: importlib not pulling resource correctly in git CI, needs to be revisited
+@pytest.mark.oss_skip
+def test_proc_mesh_bootstrap_error():
+    """
+    Test that attempts to spawn a ProcMesh with a failure during bootstrap.
+    """
+    # Run the segfault test in a subprocess
+    test_bin = importlib.resources.files("monarch.python.tests").joinpath("test_bin")
+    cmd = [
+        str(test_bin),
+        "error-bootstrap",
+    ]
+    try:
+        process = subprocess.run(cmd, capture_output=True, timeout=180)
+    except subprocess.TimeoutExpired as e:
+        print("timeout expired")
+        if e.stdout is not None:
+            print(e.stdout.decode())
+        if e.stderr is not None:
+            print(e.stderr.decode())
+        raise
 
     # Assert that the subprocess exited with a non-zero code
     assert "I actually ran" in process.stdout.decode()
     assert (
         process.returncode != 0
     ), f"Expected non-zero exit code, got {process.returncode}"
+
+
+@pytest.mark.parametrize("raise_on_getstate", [True, False])
+@pytest.mark.parametrize("raise_on_setstate", [True, False])
+@pytest.mark.parametrize("num_procs", [1, 2])
+async def test_broken_pickle_class(raise_on_getstate, raise_on_setstate, num_procs):
+    """
+    Test that exceptions during pickling/unpickling are properly handled.
+
+    This test creates a BrokenPickleClass instance configured to raise exceptions
+    during __getstate__ and/or __setstate__, then passes it to an ExceptionActor's
+    print_value endpoint and verifies that an ActorError is raised.
+    """
+    if not raise_on_getstate and not raise_on_setstate:
+        # Pass this test trivially
+        return
+
+    proc = await proc_mesh(gpus=num_procs)
+    exception_actor = await proc.spawn("exception_actor", ExceptionActor)
+
+    # Create a BrokenPickleClass instance configured to raise exceptions
+    broken_obj = BrokenPickleClass(
+        raise_on_getstate=raise_on_getstate,
+        raise_on_setstate=raise_on_setstate,
+        exception_message="Test pickle error",
+    )
+
+    # On the getstate path, we expect a RuntimeError to be raised locally.
+    # On the setstate path, we expect an ActorError to be raised remotely.
+    error_type = RuntimeError if raise_on_getstate else ActorError
+    error_pattern = "__getstate__ error" if raise_on_getstate else "__setstate__ error"
+
+    with pytest.raises(error_type, match=error_pattern):
+        if num_procs == 1:
+            await exception_actor.print_value.call_one(broken_obj)
+        else:
+            await exception_actor.print_value.call(broken_obj)

tests/test_python_actors.py

@@ -320,6 +320,11 @@ def test_sync_actor_sync_client():
     assert r == 5
 
 
+def test_proc_mesh_size() -> None:
+    proc = local_proc_mesh(gpus=2).get()
+    assert 2 == proc.size("gpus")
+
+
 def test_rank_size_sync() -> None:
     proc = local_proc_mesh(gpus=2).get()
     r = proc.spawn("runit", RunIt).get()

{torchmonarch_nightly-2025.6.6.dist-info → torchmonarch_nightly-2025.6.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: torchmonarch-nightly
-Version: 2025.6.6
+Version: 2025.6.8
 Summary: Monarch: Single controller library
 Author: Meta
 Author-email: oncall+monarch@xmail.facebook.com

{torchmonarch_nightly-2025.6.6.dist-info → torchmonarch_nightly-2025.6.8.dist-info}/RECORD

@@ -1,23 +1,23 @@
 monarch/__init__.py,sha256=iUvWHc0-7Q2tovRoRxOIiA3TsefMXCbWl-jEfQ2djew,6897
-monarch/_rust_bindings.so,sha256=b3W_9ICPY2P_yDKwRY6liTdhGGhi2euITQwhj0-4hZk,39102560
+monarch/_rust_bindings.so,sha256=HiisXwHtZrYKATL6RdJxw2u_y7Wjgjtwt52V1LIR6ss,39151608
 monarch/_testing.py,sha256=MN8DK1e-wzV0-R_nFW1b_7-O5oKfWvZ12BMGD4Z7PQk,6755
-monarch/actor_mesh.py,sha256=KfvHST97c35kyL7FslAfkRxo22UN33u3_gk2yrbZu4A,22320
+monarch/actor_mesh.py,sha256=5DbU9OrmNk5I9yasmE-rkTgHyO07oiLlAG0jbJBOXgI,23000
 monarch/allocator.py,sha256=_2DKFP9pSD33zDgH7xZJC8Tq7BQrCeQEUmMB7_xCT0Y,1784
-monarch/bootstrap_main.py,sha256=_LgEvfI_kFHj2QWH8CLRBQI1tbxS0uWrnHqwzOVbjeI,2417
+monarch/bootstrap_main.py,sha256=SYTOz-pTXiJNk78PPD5HAOJDSb8t2JfitRWdmWB3ogo,2559
 monarch/cached_remote_function.py,sha256=kYdB6r4OHx_T_uX4q3tCNcp1t2DJwF8tPTIahUiT2pU,8785
 monarch/fetch.py,sha256=61jxo7sx4QNUTkc0_rF5NaJROen4tKbAaiIjrXWLOvg,1705
 monarch/future.py,sha256=lcdFEe7m1shYPPuvZ1RkS6JUIChEKGBWe3v7x_nu4Hg,731
 monarch/gradient_generator.py,sha256=Rl3dmXGceTdCc1mYBg2JciR88ywGPnW7TVkL86KwqEA,6366
 monarch/memory.py,sha256=ol86dBhFAJqg78iF25-BuK0wuwj1onR8FIioZ_B0gjw,1377
-monarch/monarch_controller,sha256=LlUJ69r9spjCivWYdew83cFM4YszUx5Djqp9k_xxppo,20386688
+monarch/monarch_controller,sha256=5TKjcz7U7K8OttrwYv-w7yYtPUm2aMOQV4gt0u_Vj5c,20385960
 monarch/notebook.py,sha256=zu9MKDFKf1-rCM2TqFSRJjMBeiWuKcJSyUFLvoZRQzs,25949
 monarch/opaque_module.py,sha256=oajOu_WD1hD4hxE8HDdO-tvWY7KDHWd7VaAhJEa5L2I,10446
 monarch/opaque_object.py,sha256=IVpll4pyuKZMo_EnPh4s0qnx8RlAcJrJ1yoLX6E75wQ,2782
-monarch/proc_mesh.py,sha256=sTMmwQLKqM0h-yY0mn8uSzOb9B_MX9DKWCI9EsyfD6s,6384
+monarch/proc_mesh.py,sha256=pVN0BLnjGaty6-UGn1U81rNdmfiDvD4gO1c4bISHtqs,6807
 monarch/profiler.py,sha256=TQ9fnVM8H7smBWtYdB_6Irtzz8DBOmcp7U1T3wlUmco,4911
 monarch/python_local_mesh.py,sha256=YsureIzR9uGlNVrKd4vRghxOXBeYabkt9lICRErfRAI,3536
 monarch/random.py,sha256=f9QR7Esu4Vxqxs-KCf5QYyVqlWvXJ3-UtG90L_h4j40,1527
-monarch/rdma.py,sha256=eWwYKurW-Y6j68m0xH8jeyE3bfmSgB5ZwM2j-RmbCHc,6397
+monarch/rdma.py,sha256=1pNh11S_FWeETRgkdUpauTMUlodrRohIq1UfQjKVnN8,5418
 monarch/remote_class.py,sha256=-OAowzU1aDP6i4ik_SjXntVUC9h4dqAzgqwohkQ6Grc,4167
 monarch/rust_backend_mesh.py,sha256=1htC62of4MgFtkezWGlsxSFtKJdc0CIeqeSuOx7yu3M,9944
 monarch/rust_local_mesh.py,sha256=7ASptybn3wy4J7eoBc7LhGW4j4AA6bigl5Kuhyflw8s,47405
@@ -46,7 +46,7 @@ monarch/common/client.py,sha256=wOAnoaLmabrcv7mK_z_HVnk_ivGe5igPy3iWZI4LVZc,2451
 monarch/common/constants.py,sha256=ohvsVYMpfeWopv3KXDAeHWDFLukwc-OY37VRxpKNBE8,300
 monarch/common/context_manager.py,sha256=GOeyaFbyCqvQmkJ0oI7q6IxRd8_0mVyYKZRccI8iaug,1067
 monarch/common/controller_api.py,sha256=djGkK5aSd-V6pBkr3uBCXbfJv3OKf2o2VbBXJgFF2WI,3202
-monarch/common/device_mesh.py,sha256=PyVONLa0EDOzVobU-PK-mGAQyj1Dyo9dr__lDmx2uKY,13144
+monarch/common/device_mesh.py,sha256=fBZMYDpfAp5tAEXTe9l6eJxDI4-TMWVOMrAJXp5hzvI,12082
 monarch/common/fake.py,sha256=h57Cggz2qXNqImZ7yPuOZOSe9-l9i553ki1z-YHlgQA,1801
 monarch/common/function.py,sha256=V8kdgSRTvild2SpcewWa5IETX3QiWDZQ2BEIDFa5zz8,4374
 monarch/common/function_caching.py,sha256=HVdbWtv6Eea7ENMWi8iv36w1G1TaVuUJhkUX_JxGx5A,5060
@@ -63,7 +63,7 @@ monarch/common/recording.py,sha256=hoI9VY_FyW_xVx-jmfsKydqX5vW2GulwcDWsBdUVOm8,4
 monarch/common/reference.py,sha256=O26lkzEeVwj0S1xEy-OLqdHVnACmmlbQCUmXRrW4n1Q,938
 monarch/common/remote.py,sha256=qZWXkShX20l07TseQSpVECh2yXZaVKYUvQXkeEM-zvY,9220
 monarch/common/selection.py,sha256=lpWFbZs3ArYy29e-53eoAVAjQFksf1RvZz9NvM0CUW4,308
-monarch/common/shape.py,sha256=jEHneh190QI7zGOVAARpXtkxI9mXV1YbnycXlpYQGuc,7388
+monarch/common/shape.py,sha256=k6-0S0U19PmrfP62SMb9Ihx6_I4QQFUGErloZn8GcZ0,8144
 monarch/common/stream.py,sha256=J9UCqhSXSbKYFGtbKaqAq1Vgmg6DJcLzsXXm-tsBQ-w,3499
 monarch/common/tensor.py,sha256=mSXiHoD0Up4m2RLdQcsbesaz2N4QCFS34UNNX3Dbldk,28842
 monarch/common/tensor_factory.py,sha256=qm8NZx-5ezMAFjNLiXQvb66okm5XgdboB_GRarGOdN0,801
@@ -127,9 +127,9 @@ monarch_supervisor/python_executable.py,sha256=WfCiK3wdAvm9Jxx5jgjGF991NgGc9-oHU
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/dispatch_bench.py,sha256=sU_m-8KAjQgYTsxI5khV664NdgLLutidni69Rtowk98,3933
 tests/dispatch_bench_helper.py,sha256=1ORgAMrRgjAjmmWeCHLLQd_bda9mJk0rS2ucEbRu28s,633
-tests/error_test_binary.py,sha256=r9-mm4eDqaJYnBo3gXcuqwhpYq1HeH6xem3a4p8rakI,4600
+tests/error_test_binary.py,sha256=64H-ucdkQ2i7GD8sidStl227cOy7gyeqvO4kTm1y7Ic,4817
 tests/sleep_binary.py,sha256=XfLYaAfwm9xgzM-svs8fhAeFhwYIg6SyVEnx4e6wbUw,1009
-tests/test_actor_error.py,sha256=YBDS6BKwZqgKTFtydEJt4qwJGXRfWx3hgxup9ayVbhY,3827
+tests/test_actor_error.py,sha256=z3Sf4lteUggTryPLOhRKJ55v0MwVK3a7QN7-U2U9iJg,7484
 tests/test_alloc.py,sha256=D6DdQbtOZEvvnnc7LV-WyWFMk0Xb77eblH6Oz90zJTA,745
 tests/test_coalescing.py,sha256=-KtAWzTaeXbyzltplfojavx0iFeeZnvej-tFTlu2p5k,15616
 tests/test_controller.py,sha256=yxuVp2DG3TDKJlwuE3cFm9dbWMlbrYtG1uHfvVWRYbw,30935
@@ -139,7 +139,7 @@ tests/test_future.py,sha256=cXzaNi2YDwVyjR541ScXmgktX1YFsKzbl8wep0DMVbk,3032
 tests/test_grad_generator.py,sha256=p4Pm4kMEeGldt2jUVAkGKCB0mLccKI28pltH6OTGbQA,3412
 tests/test_mock_cuda.py,sha256=5hisElxeLJ5MHw3KM9gwxBiXiMaG-Rm382u3AsQcDOI,3068
 tests/test_pdb_actor.py,sha256=5KJhuhcZDPWMdjC6eAtDdwnz1W7jNFXvIrMSFaCWaPw,3858
-tests/test_python_actors.py,sha256=dY109ofFtmmni9wJWNVb3W7YQH_tMZWSIGovnuAsrUw,10786
+tests/test_python_actors.py,sha256=fDvHUIWNZeL3CWnTJMbdh98i1tnH1-LJEG1pIFkGYF8,10898
 tests/test_remote_functions.py,sha256=ExqYlRQWRabpGBuKvNIOa8Hwj-iXuP87Jfb9i5RhaGs,50066
 tests/test_rust_backend.py,sha256=nXSa0ZQ0NniZm4PzvKhrWvVLD-RKvIWYkPXm1BEBXq8,6235
 tests/test_signal_safe_block_on.py,sha256=bmal0XgzJowZXJV6T1Blow5a-vZluYWusCThLMGxyTE,3336
@@ -149,9 +149,9 @@ tests/simulator/test_profiling.py,sha256=TGYCfzTLdkpIwnOuO6KApprmrgPIRQe60KRX3wk
 tests/simulator/test_simulator.py,sha256=LO8lA0ssY-OGEBL5ipEu74f97Y765TEwfUOv-DtIptM,14568
 tests/simulator/test_task.py,sha256=ipqBDuDAysuo1xOB9S5psaFvwe6VATD43IovCTSs0t4,2327
 tests/simulator/test_worker.py,sha256=QrWWIJ3HDgDLkBPRc2mwYPlOQoXQcj1qRfc0WUfKkFY,3507
-torchmonarch_nightly-2025.6.6.dist-info/licenses/LICENSE,sha256=e0Eotbf_rHOYPuEUlppIbvwy4SN98CZnl_hqwvbDA4Q,1530
-torchmonarch_nightly-2025.6.6.dist-info/METADATA,sha256=q8PbST0aYM1zzxSGM9AjrDdDbTmJbg-7gaMyH6sqDPQ,2771
-torchmonarch_nightly-2025.6.6.dist-info/WHEEL,sha256=_wZSFk0d90K9wOBp8Q-UGxshyiJ987JoPiyUBNC6VLk,104
-torchmonarch_nightly-2025.6.6.dist-info/entry_points.txt,sha256=sqfQ16oZqjEvttUI-uj9BBXIIE6jt05bYFSmy-2hyXI,106
-torchmonarch_nightly-2025.6.6.dist-info/top_level.txt,sha256=E-ZssZzyM17glpVrh-S9--qJ-w9p2EjuYOuNw9tQ4Eg,33
-torchmonarch_nightly-2025.6.6.dist-info/RECORD,,
+torchmonarch_nightly-2025.6.8.dist-info/licenses/LICENSE,sha256=e0Eotbf_rHOYPuEUlppIbvwy4SN98CZnl_hqwvbDA4Q,1530
+torchmonarch_nightly-2025.6.8.dist-info/METADATA,sha256=AfGuuk6TyhejOLotJWjRt3Hsl80lkEWS4iOaZ61YHj4,2771
+torchmonarch_nightly-2025.6.8.dist-info/WHEEL,sha256=_wZSFk0d90K9wOBp8Q-UGxshyiJ987JoPiyUBNC6VLk,104
+torchmonarch_nightly-2025.6.8.dist-info/entry_points.txt,sha256=sqfQ16oZqjEvttUI-uj9BBXIIE6jt05bYFSmy-2hyXI,106
+torchmonarch_nightly-2025.6.8.dist-info/top_level.txt,sha256=E-ZssZzyM17glpVrh-S9--qJ-w9p2EjuYOuNw9tQ4Eg,33
+torchmonarch_nightly-2025.6.8.dist-info/RECORD,,