torchmonarch-nightly 2025.6.6__cp310-cp310-manylinux2014_x86_64.whl → 2025.6.7__cp310-cp310-manylinux2014_x86_64.whl

monarch/actor_mesh.py

@@ -39,7 +39,7 @@ from typing import (
 import monarch
 from monarch import ActorFuture as Future
 
-from monarch._rust_bindings.monarch_hyperactor.actor import PythonMessage
+from monarch._rust_bindings.monarch_hyperactor.actor import PanicFlag, PythonMessage
 from monarch._rust_bindings.monarch_hyperactor.actor_mesh import PythonActorMesh
 from monarch._rust_bindings.monarch_hyperactor.mailbox import (
     Mailbox,
@@ -462,12 +462,12 @@ class _Actor:
     def __init__(self) -> None:
         self.instance: object | None = None
         self.active_requests: asyncio.Queue[asyncio.Future[object]] = asyncio.Queue()
-        self.complete_task: object | None = None
+        self.complete_task: asyncio.Task | None = None
 
     def handle(
-        self, mailbox: Mailbox, message: PythonMessage
+        self, mailbox: Mailbox, message: PythonMessage, panic_flag: PanicFlag
     ) -> Optional[Coroutine[Any, Any, Any]]:
-        return self.handle_cast(mailbox, 0, singleton_shape, message)
+        return self.handle_cast(mailbox, 0, singleton_shape, message, panic_flag)
 
     def handle_cast(
         self,
@@ -475,6 +475,7 @@ class _Actor:
         rank: int,
         shape: Shape,
         message: PythonMessage,
+        panic_flag: PanicFlag,
     ) -> Optional[Coroutine[Any, Any, Any]]:
         port = None
         try:
@@ -495,10 +496,10 @@ class _Actor:
                         port.send("result", result)
                     return None
 
-                return self.run_async(ctx, self.run_task(port, result))
+                return self.run_async(ctx, self.run_task(port, result, panic_flag))
         except Exception as e:
             traceback.print_exc()
-            s = ActorMeshRefCallFailedException(e)
+            s = ActorError(e)
 
             # The exception is delivered to exactly one of:
             # (1) our caller, (2) our supervisor
@@ -510,17 +511,17 @@ class _Actor:
     async def run_async(self, ctx, coroutine):
         _context.set(ctx)
         if self.complete_task is None:
-            asyncio.create_task(self._complete())
+            self.complete_task = asyncio.create_task(self._complete())
         await self.active_requests.put(create_eager_task(coroutine))
 
-    async def run_task(self, port, coroutine):
+    async def run_task(self, port, coroutine, panic_flag):
         try:
             result = await coroutine
             if port is not None:
                 port.send("result", result)
         except Exception as e:
             traceback.print_exc()
-            s = ActorMeshRefCallFailedException(e)
+            s = ActorError(e)
 
             # The exception is delivered to exactly one of:
             # (1) our caller, (2) our supervisor
@@ -528,6 +529,16 @@ class _Actor:
                 port.send("exception", s)
             else:
                 raise s from None
+        except BaseException as e:
+            # A BaseException can be thrown in the case of a Rust panic.
+            # In this case, we need a way to signal the panic to the Rust side.
+            # See [Panics in async endpoints]
+            try:
+                panic_flag.signal_panic(e)
+            except Exception:
+                # The channel might be closed if the Rust side has already detected the error
+                pass
+            raise
 
     async def _complete(self) -> None:
         while True:
@@ -655,7 +666,7 @@ class ActorMeshRef(MeshTrait):
         )
 
 
-class ActorMeshRefCallFailedException(Exception):
+class ActorError(Exception):
     """
     Deterministic problem with the user's code.
     For example, an OOM resulting in trying to allocate too much GPU memory, or violating

monarch/bootstrap_main.py

@@ -53,6 +53,9 @@ def invoke_main():
                     record.levelno,
                 )
 
+    if os.environ.get("MONARCH_ERROR_DURING_BOOTSTRAP_FOR_TESTING") == "1":
+        raise RuntimeError("Error during bootstrap for testing")
+
     # forward logs to rust tracing. Defaults to on.
     if os.environ.get("MONARCH_PYTHON_LOG_TRACING", "1") == "1":
         logging.root.addHandler(TracingForwarder())

monarch/rdma.py

@@ -6,10 +6,7 @@
 
 import ctypes
 
-import traceback
-
 from dataclasses import dataclass
-from traceback import extract_tb, StackSummary
 from typing import cast, Dict, Optional, Tuple
 
 import torch
@@ -163,28 +160,3 @@ class RDMABuffer:
             src.numel(),
         )
         await RDMAManager.on_proc(self.proc_id).put.call_one(self.addr, offset, bytes)
-
-
-class ActorMeshRefCallFailedException(Exception):
-    """
-    Deterministic problem with the user's code.
-    For example, an OOM resulting in trying to allocate too much GPU memory, or violating
-    some invariant enforced by the various APIs.
-    """
-
-    def __init__(
-        self,
-        exception: Exception,
-        message: str = "A remote service call has failed asynchronously.",
-    ) -> None:
-        self.exception = exception
-        self.actor_mesh_ref_frames: StackSummary = extract_tb(exception.__traceback__)
-        self.message = message
-
-    def __str__(self) -> str:
-        exe = str(self.exception)
-        actor_mesh_ref_tb = "".join(traceback.format_list(self.actor_mesh_ref_frames))
-        return (
-            f"{self.message}\n"
-            f"Traceback of where the service call failed (most recent call last):\n{actor_mesh_ref_tb}{type(self.exception).__name__}: {exe}"
-        )

tests/error_test_binary.py

@@ -7,6 +7,8 @@
 import ctypes
 import sys
 
+import click
+
 from monarch._rust_bindings.monarch_extension.panic import panicking_function
 
 from monarch.actor_mesh import Actor, endpoint
@@ -115,24 +117,33 @@ def _run_error_test(num_procs, sync_endpoint, endpoint_name):
     asyncio.run(run_test())
 
 
+@click.group()
 def main():
-    import argparse
+    pass
 
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--num-procs", type=int)
-    parser.add_argument("--sync-test-impl", type=bool)
-    parser.add_argument("--sync-endpoint", type=bool)
-    parser.add_argument("--endpoint-name", type=str)
-    args = parser.parse_args()
 
+@main.command("error-endpoint")
+@click.option("--num-procs", type=int, required=True)
+@click.option("--sync-test-impl", type=bool, required=True)
+@click.option("--sync-endpoint", type=bool, required=True)
+@click.option("--endpoint-name", type=str, required=True)
+def error_endpoint(num_procs, sync_test_impl, sync_endpoint, endpoint_name):
     print(
-        f"Running segfault test: {args.num_procs=} {args.sync_test_impl=} {args.sync_endpoint=}, {args.endpoint_name=}"
+        f"Running segfault test: {num_procs=} {sync_test_impl=} {sync_endpoint=}, {endpoint_name=}"
     )
 
-    if args.sync_test_impl:
-        _run_error_test_sync(args.num_procs, args.sync_endpoint, args.endpoint_name)
+    if sync_test_impl:
+        _run_error_test_sync(num_procs, sync_endpoint, endpoint_name)
     else:
-        _run_error_test(args.num_procs, args.sync_endpoint, args.endpoint_name)
+        _run_error_test(num_procs, sync_endpoint, endpoint_name)
+
+
+@main.command("error-bootstrap")
+def error_bootstrap():
+    print("I actually ran")
+    sys.stdout.flush()
+
+    proc_mesh(gpus=4, env={"MONARCH_ERROR_DURING_BOOTSTRAP_FOR_TESTING": "1"}).get()
 
 
 if __name__ == "__main__":

tests/test_actor_error.py

@@ -8,47 +8,36 @@ import importlib.resources
 import subprocess
 
 import pytest
-from monarch.actor_mesh import Actor, ActorMeshRefCallFailedException, endpoint
+from monarch.actor_mesh import Actor, ActorError, endpoint
 
 from monarch.proc_mesh import proc_mesh
 
 
 class ExceptionActor(Actor):
-    """An actor that has endpoints which raise exceptions."""
-
     @endpoint
     async def raise_exception(self) -> None:
-        """Endpoint that raises an exception."""
         raise Exception("This is a test exception")
 
 
 class ExceptionActorSync(Actor):
-    """An actor that has endpoints which raise exceptions."""
-
     @endpoint  # pyre-ignore
     def raise_exception(self) -> None:
-        """Endpoint that raises an exception."""
         raise Exception("This is a test exception")
 
 
 @pytest.mark.parametrize(
-    "actor_class,actor_name",
-    [
-        (ExceptionActor, "exception_actor_async_call"),
-        (ExceptionActorSync, "exception_actor_sync_call"),
-    ],
+    "actor_class",
+    [ExceptionActor, ExceptionActorSync],
 )
 @pytest.mark.parametrize("num_procs", [1, 2])
-async def test_actor_exception(actor_class, actor_name, num_procs):
+async def test_actor_exception(actor_class, num_procs):
     """
     Test that exceptions raised in actor endpoints are propagated to the client.
     """
     proc = await proc_mesh(gpus=num_procs)
-    exception_actor = await proc.spawn(actor_name, actor_class)
+    exception_actor = await proc.spawn("exception_actor", actor_class)
 
-    with pytest.raises(
-        ActorMeshRefCallFailedException, match="This is a test exception"
-    ):
+    with pytest.raises(ActorError, match="This is a test exception"):
         if num_procs == 1:
             await exception_actor.raise_exception.call_one()
         else:
@@ -56,23 +45,18 @@ async def test_actor_exception(actor_class, actor_name, num_procs):
 
 
 @pytest.mark.parametrize(
-    "actor_class,actor_name",
-    [
-        (ExceptionActor, "exception_actor_async_call"),
-        (ExceptionActorSync, "exception_actor_sync_call"),
-    ],
+    "actor_class",
+    [ExceptionActor, ExceptionActorSync],
 )
 @pytest.mark.parametrize("num_procs", [1, 2])
-def test_actor_exception_sync(actor_class, actor_name, num_procs):
+def test_actor_exception_sync(actor_class, num_procs):
     """
     Test that exceptions raised in actor endpoints are propagated to the client.
     """
     proc = proc_mesh(gpus=num_procs).get()
-    exception_actor = proc.spawn(actor_name, actor_class).get()
+    exception_actor = proc.spawn("exception_actor", actor_class).get()
 
-    with pytest.raises(
-        ActorMeshRefCallFailedException, match="This is a test exception"
-    ):
+    with pytest.raises(ActorError, match="This is a test exception"):
         if num_procs == 1:
             exception_actor.raise_exception.call_one().get()
         else:
@@ -85,25 +69,62 @@ def test_actor_exception_sync(actor_class, actor_name, num_procs):
 @pytest.mark.parametrize("sync_endpoint", [False, True])
 @pytest.mark.parametrize("sync_test_impl", [False, True])
 @pytest.mark.parametrize("endpoint_name", ["cause_segfault", "cause_panic"])
-def test_actor_segfault(num_procs, sync_endpoint, sync_test_impl, endpoint_name):
+def test_actor_supervision(num_procs, sync_endpoint, sync_test_impl, endpoint_name):
     """
-    Test that segfaults in actor endpoints result in a non-zero exit code.
-    This test spawns a subprocess that will segfault and checks its exit code.
+    Test that an endpoint causing spontaenous process exit is handled by the supervisor.
 
-    Tests both ExceptionActor and ExceptionActorSync using async API.
+    Today, these events are delivered to the client and cause the client process
+    to exit with a non-zero code, so the only way we can test it is via a
+    subprocess harness.
     """
     # Run the segfault test in a subprocess
     test_bin = importlib.resources.files("monarch.python.tests").joinpath("test_bin")
     cmd = [
         str(test_bin),
+        "error-endpoint",
         f"--num-procs={num_procs}",
         f"--sync-endpoint={sync_endpoint}",
         f"--sync-test-impl={sync_test_impl}",
         f"--endpoint-name={endpoint_name}",
     ]
-    process = subprocess.run(cmd, capture_output=True, timeout=60)
-    print(process.stdout.decode())
-    print(process.stderr.decode())
+    try:
+        process = subprocess.run(cmd, capture_output=True, timeout=180)
+    except subprocess.TimeoutExpired as e:
+        print("timeout expired")
+        if e.stdout is not None:
+            print(e.stdout.decode())
+        if e.stderr is not None:
+            print(e.stderr.decode())
+        raise
+
+    # Assert that the subprocess exited with a non-zero code
+    assert "I actually ran" in process.stdout.decode()
+    assert (
+        process.returncode != 0
+    ), f"Expected non-zero exit code, got {process.returncode}"
+
+
+# oss_skip: importlib not pulling resource correctly in git CI, needs to be revisited
+@pytest.mark.oss_skip
+def test_proc_mesh_bootstrap_error():
+    """
+    Test that attempts to spawn a ProcMesh with a failure during bootstrap.
+    """
+    # Run the segfault test in a subprocess
+    test_bin = importlib.resources.files("monarch.python.tests").joinpath("test_bin")
+    cmd = [
+        str(test_bin),
+        "error-bootstrap",
+    ]
+    try:
+        process = subprocess.run(cmd, capture_output=True, timeout=180)
+    except subprocess.TimeoutExpired as e:
+        print("timeout expired")
+        if e.stdout is not None:
+            print(e.stdout.decode())
+        if e.stderr is not None:
+            print(e.stderr.decode())
+        raise
 
     # Assert that the subprocess exited with a non-zero code
     assert "I actually ran" in process.stdout.decode()

{torchmonarch_nightly-2025.6.6.dist-info → torchmonarch_nightly-2025.6.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: torchmonarch-nightly
-Version: 2025.6.6
+Version: 2025.6.7
 Summary: Monarch: Single controller library
 Author: Meta
 Author-email: oncall+monarch@xmail.facebook.com

{torchmonarch_nightly-2025.6.6.dist-info → torchmonarch_nightly-2025.6.7.dist-info}/RECORD

@@ -1,15 +1,15 @@
 monarch/__init__.py,sha256=iUvWHc0-7Q2tovRoRxOIiA3TsefMXCbWl-jEfQ2djew,6897
-monarch/_rust_bindings.so,sha256=b3W_9ICPY2P_yDKwRY6liTdhGGhi2euITQwhj0-4hZk,39102560
+monarch/_rust_bindings.so,sha256=seyf4m6FoGBVbC4DBiG2dJdoSqsVRTWTbH9KeBvc1Is,39128520
 monarch/_testing.py,sha256=MN8DK1e-wzV0-R_nFW1b_7-O5oKfWvZ12BMGD4Z7PQk,6755
-monarch/actor_mesh.py,sha256=KfvHST97c35kyL7FslAfkRxo22UN33u3_gk2yrbZu4A,22320
+monarch/actor_mesh.py,sha256=_IVpdQ1HIC5JumB7cwcamdeWREPKqeMYjiz7NOV9Klw,22842
 monarch/allocator.py,sha256=_2DKFP9pSD33zDgH7xZJC8Tq7BQrCeQEUmMB7_xCT0Y,1784
-monarch/bootstrap_main.py,sha256=_LgEvfI_kFHj2QWH8CLRBQI1tbxS0uWrnHqwzOVbjeI,2417
+monarch/bootstrap_main.py,sha256=SYTOz-pTXiJNk78PPD5HAOJDSb8t2JfitRWdmWB3ogo,2559
 monarch/cached_remote_function.py,sha256=kYdB6r4OHx_T_uX4q3tCNcp1t2DJwF8tPTIahUiT2pU,8785
 monarch/fetch.py,sha256=61jxo7sx4QNUTkc0_rF5NaJROen4tKbAaiIjrXWLOvg,1705
 monarch/future.py,sha256=lcdFEe7m1shYPPuvZ1RkS6JUIChEKGBWe3v7x_nu4Hg,731
 monarch/gradient_generator.py,sha256=Rl3dmXGceTdCc1mYBg2JciR88ywGPnW7TVkL86KwqEA,6366
 monarch/memory.py,sha256=ol86dBhFAJqg78iF25-BuK0wuwj1onR8FIioZ_B0gjw,1377
-monarch/monarch_controller,sha256=LlUJ69r9spjCivWYdew83cFM4YszUx5Djqp9k_xxppo,20386688
+monarch/monarch_controller,sha256=ptqqARjqzjjVP0R-1ZPAd0y_K1-0XHFQhE-HR4J6MOo,20389704
 monarch/notebook.py,sha256=zu9MKDFKf1-rCM2TqFSRJjMBeiWuKcJSyUFLvoZRQzs,25949
 monarch/opaque_module.py,sha256=oajOu_WD1hD4hxE8HDdO-tvWY7KDHWd7VaAhJEa5L2I,10446
 monarch/opaque_object.py,sha256=IVpll4pyuKZMo_EnPh4s0qnx8RlAcJrJ1yoLX6E75wQ,2782
@@ -17,7 +17,7 @@ monarch/proc_mesh.py,sha256=sTMmwQLKqM0h-yY0mn8uSzOb9B_MX9DKWCI9EsyfD6s,6384
 monarch/profiler.py,sha256=TQ9fnVM8H7smBWtYdB_6Irtzz8DBOmcp7U1T3wlUmco,4911
 monarch/python_local_mesh.py,sha256=YsureIzR9uGlNVrKd4vRghxOXBeYabkt9lICRErfRAI,3536
 monarch/random.py,sha256=f9QR7Esu4Vxqxs-KCf5QYyVqlWvXJ3-UtG90L_h4j40,1527
-monarch/rdma.py,sha256=eWwYKurW-Y6j68m0xH8jeyE3bfmSgB5ZwM2j-RmbCHc,6397
+monarch/rdma.py,sha256=1pNh11S_FWeETRgkdUpauTMUlodrRohIq1UfQjKVnN8,5418
 monarch/remote_class.py,sha256=-OAowzU1aDP6i4ik_SjXntVUC9h4dqAzgqwohkQ6Grc,4167
 monarch/rust_backend_mesh.py,sha256=1htC62of4MgFtkezWGlsxSFtKJdc0CIeqeSuOx7yu3M,9944
 monarch/rust_local_mesh.py,sha256=7ASptybn3wy4J7eoBc7LhGW4j4AA6bigl5Kuhyflw8s,47405
@@ -127,9 +127,9 @@ monarch_supervisor/python_executable.py,sha256=WfCiK3wdAvm9Jxx5jgjGF991NgGc9-oHU
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/dispatch_bench.py,sha256=sU_m-8KAjQgYTsxI5khV664NdgLLutidni69Rtowk98,3933
 tests/dispatch_bench_helper.py,sha256=1ORgAMrRgjAjmmWeCHLLQd_bda9mJk0rS2ucEbRu28s,633
-tests/error_test_binary.py,sha256=r9-mm4eDqaJYnBo3gXcuqwhpYq1HeH6xem3a4p8rakI,4600
+tests/error_test_binary.py,sha256=64H-ucdkQ2i7GD8sidStl227cOy7gyeqvO4kTm1y7Ic,4817
 tests/sleep_binary.py,sha256=XfLYaAfwm9xgzM-svs8fhAeFhwYIg6SyVEnx4e6wbUw,1009
-tests/test_actor_error.py,sha256=YBDS6BKwZqgKTFtydEJt4qwJGXRfWx3hgxup9ayVbhY,3827
+tests/test_actor_error.py,sha256=K4buy0Z3MfCF7uSgIMRCpw7A2fTl3iRh8g_aNiJHnBU,4530
 tests/test_alloc.py,sha256=D6DdQbtOZEvvnnc7LV-WyWFMk0Xb77eblH6Oz90zJTA,745
 tests/test_coalescing.py,sha256=-KtAWzTaeXbyzltplfojavx0iFeeZnvej-tFTlu2p5k,15616
 tests/test_controller.py,sha256=yxuVp2DG3TDKJlwuE3cFm9dbWMlbrYtG1uHfvVWRYbw,30935
@@ -149,9 +149,9 @@ tests/simulator/test_profiling.py,sha256=TGYCfzTLdkpIwnOuO6KApprmrgPIRQe60KRX3wk
 tests/simulator/test_simulator.py,sha256=LO8lA0ssY-OGEBL5ipEu74f97Y765TEwfUOv-DtIptM,14568
 tests/simulator/test_task.py,sha256=ipqBDuDAysuo1xOB9S5psaFvwe6VATD43IovCTSs0t4,2327
 tests/simulator/test_worker.py,sha256=QrWWIJ3HDgDLkBPRc2mwYPlOQoXQcj1qRfc0WUfKkFY,3507
-torchmonarch_nightly-2025.6.6.dist-info/licenses/LICENSE,sha256=e0Eotbf_rHOYPuEUlppIbvwy4SN98CZnl_hqwvbDA4Q,1530
-torchmonarch_nightly-2025.6.6.dist-info/METADATA,sha256=q8PbST0aYM1zzxSGM9AjrDdDbTmJbg-7gaMyH6sqDPQ,2771
-torchmonarch_nightly-2025.6.6.dist-info/WHEEL,sha256=_wZSFk0d90K9wOBp8Q-UGxshyiJ987JoPiyUBNC6VLk,104
-torchmonarch_nightly-2025.6.6.dist-info/entry_points.txt,sha256=sqfQ16oZqjEvttUI-uj9BBXIIE6jt05bYFSmy-2hyXI,106
-torchmonarch_nightly-2025.6.6.dist-info/top_level.txt,sha256=E-ZssZzyM17glpVrh-S9--qJ-w9p2EjuYOuNw9tQ4Eg,33
-torchmonarch_nightly-2025.6.6.dist-info/RECORD,,
+torchmonarch_nightly-2025.6.7.dist-info/licenses/LICENSE,sha256=e0Eotbf_rHOYPuEUlppIbvwy4SN98CZnl_hqwvbDA4Q,1530
+torchmonarch_nightly-2025.6.7.dist-info/METADATA,sha256=8082irkhKa1D8z8Dq0GBZAfdQXh_JXcirAjEAI-A2_8,2771
+torchmonarch_nightly-2025.6.7.dist-info/WHEEL,sha256=_wZSFk0d90K9wOBp8Q-UGxshyiJ987JoPiyUBNC6VLk,104
+torchmonarch_nightly-2025.6.7.dist-info/entry_points.txt,sha256=sqfQ16oZqjEvttUI-uj9BBXIIE6jt05bYFSmy-2hyXI,106
+torchmonarch_nightly-2025.6.7.dist-info/top_level.txt,sha256=E-ZssZzyM17glpVrh-S9--qJ-w9p2EjuYOuNw9tQ4Eg,33
+torchmonarch_nightly-2025.6.7.dist-info/RECORD,,