torchmonarch-nightly 2025.6.27__cp312-cp312-manylinux2014_x86_64.whl → 2025.6.29__cp312-cp312-manylinux2014_x86_64.whl

monarch/mesh_controller.py

@@ -202,7 +202,6 @@ def _initialize_env(worker_point: Point, proc_id: str) -> None:
         num_worker_procs = len(worker_point.shape)
         process_env = {
             **worker_env,
-            "HYPERACTOR_MANAGED_SUBPROCESS": "1",
             "CUDA_VISIBLE_DEVICES": str(local_rank),
             "NCCL_HOSTID": f"{proc_id}_host_{worker_rank // gpus_per_host}",
             # This is needed to avoid a hard failure in ncclx when we do not

monarch/proc_mesh.py

@@ -38,7 +38,10 @@ from monarch._rust_bindings.hyperactor_extension.alloc import (  # @manual=//mon
     AllocSpec,
 )
 from monarch._rust_bindings.monarch_hyperactor.mailbox import Mailbox
-from monarch._rust_bindings.monarch_hyperactor.proc_mesh import ProcMesh as HyProcMesh
+from monarch._rust_bindings.monarch_hyperactor.proc_mesh import (
+    ProcMesh as HyProcMesh,
+    ProcMeshMonitor,
+)
 from monarch._rust_bindings.monarch_hyperactor.shape import Shape, Slice
 from monarch.actor_mesh import _Actor, _ActorMeshRefImpl, Actor, ActorMeshRef
 
@@ -117,6 +120,24 @@ class ProcMesh(MeshTrait):
             lambda: self._spawn_blocking(name, Class, *args, **kwargs),
         )
 
+    async def monitor(self) -> ProcMeshMonitor:
+        """
+        Get a monitor (async iterator) of the proc mesh, it is used to
+        monitor the status of the proc mesh. This function can be called at most once.
+
+        Note: This API is experimental and subject to change.
+
+        Example:
+
+        async def monitor_loop(monitor):
+            async for event in monitor:
+                await handle_exception_event(event)
+
+        # Kick off in background
+        asyncio.create_task(monitor_loop(monitor))
+        """
+        return await self._proc_mesh.monitor()
+
     @classmethod
     def from_alloc(self, alloc: Alloc) -> Future["ProcMesh"]:
         return Future(
@@ -269,7 +290,6 @@ async def proc_mesh_nonblocking(
     env = env or {}
     cmd, args, base_env = _get_bootstrap_args()
     env.update(base_env)
-    env["HYPERACTOR_MANAGED_SUBPROCESS"] = "1"
     allocator = monarch.ProcessAllocator(cmd, args, env)
     alloc = await allocator.allocate(spec)
     return await ProcMesh.from_alloc(alloc)
@@ -284,7 +304,6 @@ def proc_mesh_blocking(
     env = env or {}
     cmd, args, base_env = _get_bootstrap_args()
     env.update(base_env)
-    env["HYPERACTOR_MANAGED_SUBPROCESS"] = "1"
     allocator = monarch.ProcessAllocator(cmd, args, env)
     alloc = allocator.allocate(spec).get()
     return ProcMesh.from_alloc(alloc).get()

monarch/rust_local_mesh.py

@@ -117,9 +117,7 @@ class ControllerParams(NamedTuple):
     fail_on_worker_timeout: bool
 
 
-_PROC_ENV = {
-    "HYPERACTOR_MANAGED_SUBPROCESS": str(1),
-}
+_PROC_ENV: dict[str, str] = {}
 
 
 def get_controller_main() -> tuple[Path, dict[str, str]]:
@@ -988,7 +986,6 @@ class Bootstrap:
             raise ValueError(f"Unknown socket type: {socket_type}")
 
         env = os.environ.copy()
-        env["HYPERACTOR_MANAGED_SUBPROCESS"] = "1"
         self.env: dict[str, str] = env
 
         # Launch a single system globally

monarch/sim_mesh.py

@@ -194,7 +194,6 @@ class Bootstrap:
         fake_call(lambda: 0)
 
         env = os.environ.copy()
-        env["HYPERACTOR_MANAGED_SUBPROCESS"] = "1"
         self.env: dict[str, str] = env
 
         self._mesh_world_state: Dict[MeshWorld, Optional[DeviceMesh]] = mesh_world_state

tests/test_actor_error.py

@@ -9,9 +9,9 @@ import importlib.resources
 import subprocess
 
 import pytest
+from monarch._rust_bindings.monarch_hyperactor.proc_mesh import ProcEvent
 from monarch.actor_mesh import Actor, ActorError, endpoint, send
-
-from monarch.proc_mesh import proc_mesh
+from monarch.proc_mesh import local_proc_mesh, proc_mesh
 
 
 class ExceptionActor(Actor):
@@ -238,3 +238,36 @@ async def test_exception_after_wait_unmonitored():
     assert (
         process.returncode != 0
     ), f"Expected non-zero exit code, got {process.returncode}"
+
+
+class ErrorActor(Actor):
+    def __init__(self, message):
+        raise RuntimeError("fail on init")
+
+    @endpoint
+    async def check(self) -> None:
+        pass
+
+
+async def test_proc_mesh_redundant_monitoring():
+    proc = await local_proc_mesh(hosts=1, gpus=1)
+    await proc.monitor()
+
+    with pytest.raises(
+        Exception, match="user already registered a monitor for this proc mesh"
+    ):
+        await proc.monitor()
+
+
+async def test_proc_mesh_monitoring():
+    proc = await local_proc_mesh(hosts=1, gpus=1)
+    monitor = await proc.monitor()
+
+    with pytest.raises(Exception):
+        e = await proc.spawn("error", ErrorActor, "failed to init the actor")
+        await asyncio.wait_for(e.check.call_one(), timeout=15)
+
+    event = await anext(monitor)
+    assert isinstance(event, ProcEvent.Crashed)
+    assert event[0] == 0  # check rank
+    assert "fail on init" in event[1]  # check error message

{torchmonarch_nightly-2025.6.27.dist-info → torchmonarch_nightly-2025.6.29.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: torchmonarch-nightly
-Version: 2025.6.27
+Version: 2025.6.29
 Summary: Monarch: Single controller library
 Author: Meta
 Author-email: oncall+monarch@xmail.facebook.com

{torchmonarch_nightly-2025.6.27.dist-info → torchmonarch_nightly-2025.6.29.dist-info}/RECORD

@@ -1,5 +1,5 @@
 monarch/__init__.py,sha256=iUvWHc0-7Q2tovRoRxOIiA3TsefMXCbWl-jEfQ2djew,6897
-monarch/_rust_bindings.so,sha256=oMDFcOj81KxjZCQCBum3L1giy97jfyGw4fXdpix-xS4,43392968
+monarch/_rust_bindings.so,sha256=120L31EpmFrwc2rDVm_8lOry_zNeAcdU91Au78_yrmc,43743920
 monarch/_testing.py,sha256=jOIOG6jcZBzvEvG_DwSnwCkaMVXvSun6sJAG6nXemww,7859
 monarch/actor_mesh.py,sha256=QqKHVTJk9H_I-v7GoxgOdOL8-ymnRpGvNFdda0-cNrE,24534
 monarch/allocator.py,sha256=l0_mN43AH3K2aCchb5fk8ml95rvdgR31nRC_PqRmZWg,7865
@@ -11,21 +11,21 @@ monarch/fetch.py,sha256=61jxo7sx4QNUTkc0_rF5NaJROen4tKbAaiIjrXWLOvg,1705
 monarch/future.py,sha256=g1VYJl8ReBBS6VbikwWilnFqEr5qJDiSKid92AnWFV4,2058
 monarch/gradient_generator.py,sha256=Rl3dmXGceTdCc1mYBg2JciR88ywGPnW7TVkL86KwqEA,6366
 monarch/memory.py,sha256=ol86dBhFAJqg78iF25-BuK0wuwj1onR8FIioZ_B0gjw,1377
-monarch/mesh_controller.py,sha256=am1QP7dvn0OH1z9ADSKm41APs1HY_dHcBAhOVP-QDmE,10427
-monarch/monarch_controller,sha256=a28zj3g-ze3sihp4xppXnCfQAXrjPD92X_EwTxPpT9c,21678216
+monarch/mesh_controller.py,sha256=JIg2-MIGhqeFCXn793j8ivEu5-8ePksu--H_Pv0-Ltk,10377
+monarch/monarch_controller,sha256=sUs7I26272aWQdarhsmG1rER5EBhe8FEALbNXmzkVVI,21723248
 monarch/notebook.py,sha256=zu9MKDFKf1-rCM2TqFSRJjMBeiWuKcJSyUFLvoZRQzs,25949
 monarch/opaque_module.py,sha256=oajOu_WD1hD4hxE8HDdO-tvWY7KDHWd7VaAhJEa5L2I,10446
 monarch/opaque_object.py,sha256=IVpll4pyuKZMo_EnPh4s0qnx8RlAcJrJ1yoLX6E75wQ,2782
 monarch/pdb_wrapper.py,sha256=gm46AZnfR4amH1vYFWnWivEv5MaU3Nb6KIWjSM8KjWM,4052
-monarch/proc_mesh.py,sha256=ZnNWjINoFTdkRVbu_ikos2jV4Ham-I9jqeWdEN-1ZtQ,10436
+monarch/proc_mesh.py,sha256=UX8qthL0RSPwOf7I5dLHejVAQrZtYAERGhBeUDR4Xfw,10950
 monarch/profiler.py,sha256=TQ9fnVM8H7smBWtYdB_6Irtzz8DBOmcp7U1T3wlUmco,4911
 monarch/python_local_mesh.py,sha256=YsureIzR9uGlNVrKd4vRghxOXBeYabkt9lICRErfRAI,3536
 monarch/random.py,sha256=f9QR7Esu4Vxqxs-KCf5QYyVqlWvXJ3-UtG90L_h4j40,1527
 monarch/rdma.py,sha256=1pNh11S_FWeETRgkdUpauTMUlodrRohIq1UfQjKVnN8,5418
 monarch/remote_class.py,sha256=-OAowzU1aDP6i4ik_SjXntVUC9h4dqAzgqwohkQ6Grc,4167
 monarch/rust_backend_mesh.py,sha256=1htC62of4MgFtkezWGlsxSFtKJdc0CIeqeSuOx7yu3M,9944
-monarch/rust_local_mesh.py,sha256=7ASptybn3wy4J7eoBc7LhGW4j4AA6bigl5Kuhyflw8s,47405
-monarch/sim_mesh.py,sha256=kDsbubv28YFg9ZQN4urt3oJGzR3CnnUiATnjUiSxrkE,12256
+monarch/rust_local_mesh.py,sha256=hcVBlX73UZkXBZM2AzV2Fp01ftM5WRNc5LVV81LFMr8,47324
+monarch/sim_mesh.py,sha256=omx_dmHOjPQqj15PgphN-OOzSgfZ9MllOWhO_AYW-8U,12205
 monarch/telemetry.py,sha256=7JUZWaoD2Yn5Ae_7kNhkLFRBLYaSGfH071_m_qfVehI,525
 monarch/tensor_worker_main.py,sha256=Nbarl2sJKIddLeaRFsaUnqOerLHjzggUr9SqCr2_GYI,8300
 monarch/tensorboard.py,sha256=MnLgH5lbqeUJauEuirEgR6L_qYl2NGdtwZOWIAuOZao,2587
@@ -135,7 +135,7 @@ tests/dispatch_bench.py,sha256=sU_m-8KAjQgYTsxI5khV664NdgLLutidni69Rtowk98,3933
 tests/dispatch_bench_helper.py,sha256=1ORgAMrRgjAjmmWeCHLLQd_bda9mJk0rS2ucEbRu28s,633
 tests/error_test_binary.py,sha256=BRj13wAROsUWx4jcxc07HYN2n-xyBNhnnRAhjqah-A0,5582
 tests/sleep_binary.py,sha256=XfLYaAfwm9xgzM-svs8fhAeFhwYIg6SyVEnx4e6wbUw,1009
-tests/test_actor_error.py,sha256=-0UJCEpyzsBh-RdbGhDiG1-sRtu7bJPQWmtjUD0ad48,8526
+tests/test_actor_error.py,sha256=hMpaJDkvowBM_fKuitQdKmq3iyvySaom8KNLmxEsZ-o,9534
 tests/test_alloc.py,sha256=D6DdQbtOZEvvnnc7LV-WyWFMk0Xb77eblH6Oz90zJTA,745
 tests/test_allocator.py,sha256=c7b4ylEjFV2WDhB8fbWiDuGi-vrBeD1E0Rpu-efrSVQ,14478
 tests/test_coalescing.py,sha256=JZ4YgQNlWWs7N-Z8KCCXQPANcuyyXEKjeHIXYbPnQhk,15606
@@ -157,9 +157,9 @@ tests/simulator/test_profiling.py,sha256=TGYCfzTLdkpIwnOuO6KApprmrgPIRQe60KRX3wk
 tests/simulator/test_simulator.py,sha256=LO8lA0ssY-OGEBL5ipEu74f97Y765TEwfUOv-DtIptM,14568
 tests/simulator/test_task.py,sha256=ipqBDuDAysuo1xOB9S5psaFvwe6VATD43IovCTSs0t4,2327
 tests/simulator/test_worker.py,sha256=QrWWIJ3HDgDLkBPRc2mwYPlOQoXQcj1qRfc0WUfKkFY,3507
-torchmonarch_nightly-2025.6.27.dist-info/licenses/LICENSE,sha256=e0Eotbf_rHOYPuEUlppIbvwy4SN98CZnl_hqwvbDA4Q,1530
-torchmonarch_nightly-2025.6.27.dist-info/METADATA,sha256=0PKqq2myfJJjhPa9nAZVJCp4vymD0dBmC1w-RmRKgYI,2780
-torchmonarch_nightly-2025.6.27.dist-info/WHEEL,sha256=lduYNUEDASmtUEDemd8SmeX1qOMvvA6YKAbAo1Qbwk8,104
-torchmonarch_nightly-2025.6.27.dist-info/entry_points.txt,sha256=sqfQ16oZqjEvttUI-uj9BBXIIE6jt05bYFSmy-2hyXI,106
-torchmonarch_nightly-2025.6.27.dist-info/top_level.txt,sha256=E-ZssZzyM17glpVrh-S9--qJ-w9p2EjuYOuNw9tQ4Eg,33
-torchmonarch_nightly-2025.6.27.dist-info/RECORD,,
+torchmonarch_nightly-2025.6.29.dist-info/licenses/LICENSE,sha256=e0Eotbf_rHOYPuEUlppIbvwy4SN98CZnl_hqwvbDA4Q,1530
+torchmonarch_nightly-2025.6.29.dist-info/METADATA,sha256=pfxXahOAFVZWSQpbC-6VgvxpOnSxa8Acic1RsONWV6U,2780
+torchmonarch_nightly-2025.6.29.dist-info/WHEEL,sha256=lduYNUEDASmtUEDemd8SmeX1qOMvvA6YKAbAo1Qbwk8,104
+torchmonarch_nightly-2025.6.29.dist-info/entry_points.txt,sha256=sqfQ16oZqjEvttUI-uj9BBXIIE6jt05bYFSmy-2hyXI,106
+torchmonarch_nightly-2025.6.29.dist-info/top_level.txt,sha256=E-ZssZzyM17glpVrh-S9--qJ-w9p2EjuYOuNw9tQ4Eg,33
+torchmonarch_nightly-2025.6.29.dist-info/RECORD,,