torchmonarch-nightly 2025.6.16__cp310-cp310-manylinux2014_x86_64.whl → 2025.6.18__cp310-cp310-manylinux2014_x86_64.whl

monarch/actor_mesh.py

@@ -6,7 +6,6 @@
 
 # pyre-unsafe
 
-import asyncio
 import collections
 import contextvars
 import functools
@@ -27,9 +26,7 @@ from typing import (
     Callable,
     cast,
     Concatenate,
-    Coroutine,
     Dict,
-    Generator,
     Generic,
     Iterable,
     List,
@@ -99,39 +96,6 @@ _context: contextvars.ContextVar[MonarchContext] = contextvars.ContextVar(
 )
 
 
-# this was implemented in python 3.12 as an argument to task
-# but I have to backport to 3.10/3.11.
-def create_eager_task(coro: Awaitable[None]) -> asyncio.Future:
-    iter = coro.__await__()
-    try:
-        first_yield = next(iter)
-        return asyncio.create_task(RestOfCoroutine(first_yield, iter).run())
-    except StopIteration as e:
-        t = asyncio.Future()
-        t.set_result(e.value)
-        return t
-
-
-class RestOfCoroutine(Generic[T1, T2]):
-    def __init__(self, first_yield: T1, iter: Generator[T2, None, T2]) -> None:
-        self.first_yield: T1 | None = first_yield
-        self.iter: Generator[T2, None, T2] = iter
-
-    def __await__(self) -> Generator[T1, None, T1] | Generator[T2, None, T2]:
-        first_yield = self.first_yield
-        assert first_yield is not None
-        yield first_yield
-        self.first_yield = None
-        while True:
-            try:
-                yield next(self.iter)
-            except StopIteration as e:
-                return e.value
-
-    async def run(self) -> T1 | T2:
-        return await self
-
-
 T = TypeVar("T")
 P = ParamSpec("P")
 R = TypeVar("R")
@@ -285,7 +249,18 @@ class Endpoint(Generic[P, R]):
         async def process() -> ValueMesh[R]:
             results: List[R] = [None] * len(self._actor_mesh)  # pyre-fixme[9]
             for _ in range(len(self._actor_mesh)):
-                rank, value = await r.recv()  # pyre-fixme[23]
+                rank, value = await r.recv()
+                results[rank] = value
+            call_shape = Shape(
+                self._actor_mesh._shape.labels,
+                NDSlice.new_row_major(self._actor_mesh._shape.ndslice.sizes),
+            )
+            return ValueMesh(call_shape, results)
+
+        def process_blocking() -> ValueMesh[R]:
+            results: List[R] = [None] * len(self._actor_mesh)  # pyre-fixme[9]
+            for _ in range(len(self._actor_mesh)):
+                rank, value = r.recv().get()
                 results[rank] = value
             call_shape = Shape(
                 self._actor_mesh._shape.labels,
@@ -293,7 +268,7 @@ class Endpoint(Generic[P, R]):
             )
             return ValueMesh(call_shape, results)
 
-        return Future(process)
+        return Future(process, process_blocking)
 
     async def stream(self, *args: P.args, **kwargs: P.kwargs) -> AsyncGenerator[R, R]:
         """
@@ -362,6 +337,9 @@ class ValueMesh(MeshTrait, Generic[R]):
     def __len__(self) -> int:
         return len(self._shape)
 
+    def __repr__(self) -> str:
+        return f"ValueMesh({self._shape})"
+
     @property
     def _ndslice(self) -> NDSlice:
         return self._shape.ndslice
@@ -485,24 +463,36 @@ singleton_shape = Shape([], NDSlice(offset=0, sizes=[], strides=[]))
 
 
 class _Actor:
+    """
+    This is the message handling implementation of a Python actor.
+
+    The layering goes:
+        Rust `PythonActor` -> `_Actor` -> user-provided `Actor` instance
+
+    Messages are received from the Rust backend, and forwarded to the `handle`
+    methods on this class.
+
+    This class wraps the actual `Actor` instance provided by the user, and
+    routes messages to it, managing argument serialization/deserialization and
+    error handling.
+    """
+
     def __init__(self) -> None:
         self.instance: object | None = None
-        self.active_requests: asyncio.Queue[asyncio.Future[object]] = asyncio.Queue()
-        self.complete_task: asyncio.Task | None = None
 
-    def handle(
+    async def handle(
         self, mailbox: Mailbox, message: PythonMessage, panic_flag: PanicFlag
-    ) -> Optional[Coroutine[Any, Any, Any]]:
-        return self.handle_cast(mailbox, 0, singleton_shape, message, panic_flag)
+    ) -> None:
+        return await self.handle_cast(mailbox, 0, singleton_shape, message, panic_flag)
 
-    def handle_cast(
+    async def handle_cast(
         self,
         mailbox: Mailbox,
         rank: int,
         shape: Shape,
         message: PythonMessage,
         panic_flag: PanicFlag,
-    ) -> Optional[Coroutine[Any, Any, Any]]:
+    ) -> None:
         port = (
             Port(message.response_port, mailbox, rank)
             if message.response_port
@@ -515,26 +505,21 @@ class _Actor:
             _context.set(ctx)
 
             args, kwargs = _unpickle(message.message, mailbox)
+
             if message.method == "__init__":
                 Class, *args = args
                 self.instance = Class(*args, **kwargs)
                 return None
-            else:
-                the_method = getattr(self.instance, message.method)._method
 
-                if not inspect.iscoroutinefunction(the_method):
-                    enter_span(
-                        the_method.__module__, message.method, str(ctx.mailbox.actor_id)
-                    )
-                    result = the_method(self.instance, *args, **kwargs)
-                    exit_span()
-                    if port is not None:
-                        port.send("result", result)
-                    return None
+            the_method = getattr(self.instance, message.method)._method
+
+            if inspect.iscoroutinefunction(the_method):
 
                 async def instrumented():
                     enter_span(
-                        the_method.__module__, message.method, str(ctx.mailbox.actor_id)
+                        the_method.__module__,
+                        message.method,
+                        str(ctx.mailbox.actor_id),
                     )
                     try:
                         result = await the_method(self.instance, *args, **kwargs)
@@ -547,39 +532,14 @@ class _Actor:
                     exit_span()
                     return result
 
-                return self.run_async(
-                    ctx,
-                    self.run_task(port, instrumented(), panic_flag),
-                )
-        except Exception as e:
-            traceback.print_exc()
-            s = ActorError(e)
-
-            # The exception is delivered to exactly one of:
-            # (1) our caller, (2) our supervisor
-            if port is not None:
-                port.send("exception", s)
+                result = await instrumented()
             else:
-                raise s from None
-
-    async def run_async(
-        self,
-        ctx: MonarchContext,
-        coroutine: Awaitable[None],
-    ) -> None:
-        _context.set(ctx)
-        if self.complete_task is None:
-            self.complete_task = asyncio.create_task(self._complete())
-        await self.active_requests.put(create_eager_task(coroutine))
+                enter_span(
+                    the_method.__module__, message.method, str(ctx.mailbox.actor_id)
+                )
+                result = the_method(self.instance, *args, **kwargs)
+                exit_span()
 
-    async def run_task(
-        self,
-        port: Port | None,
-        coroutine: Awaitable[Any],
-        panic_flag: PanicFlag,
-    ) -> None:
-        try:
-            result = await coroutine
             if port is not None:
                 port.send("result", result)
         except Exception as e:
@@ -603,11 +563,6 @@ class _Actor:
                 pass
             raise
 
-    async def _complete(self) -> None:
-        while True:
-            task = await self.active_requests.get()
-            await task
-
 
 def _is_mailbox(x: object) -> bool:
     return isinstance(x, Mailbox)
@@ -648,8 +603,8 @@ class Actor(MeshTrait):
             "actor implementations are not meshes, but we can't convince the typechecker of it..."
         )
 
-    @endpoint
-    async def _set_debug_client(self, client: "DebugClient") -> None:
+    @endpoint  # pyre-ignore
+    def _set_debug_client(self, client: "DebugClient") -> None:
         point = MonarchContext.get().point
         # For some reason, using a lambda instead of functools.partial
         # confuses the pdb wrapper implementation.
@@ -750,6 +705,9 @@ class ActorMeshRef(MeshTrait):
             self._mailbox,
         )
 
+    def __repr__(self) -> str:
+        return f"ActorMeshRef(class={self._class}, shape={self._actor_mesh_ref._shape})"
+
 
 class ActorError(Exception):
     """

monarch/common/device_mesh.py

@@ -244,24 +244,24 @@ class DeviceMesh(Referenceable, MeshTrait):
     def rotate(self, **kwargs: Dict[str, int]):
         raise NotImplementedError()
 
-    def rank(self, dims: Union[str, Sequence[str]]) -> int:
+    def rank(self, dims: Union[str, Sequence[str]]) -> torch.Tensor:
         self.define_remotely()
         if isinstance(dims, str):
             if dims not in self.names:
                 raise KeyError(f"{self} does not have dimension {repr(dims)}")
             return _remote(
-                "monarch.worker.worker._rank",
+                _rank,
                 propagate=lambda _self, _dims: torch.full((), 0, dtype=torch.long),
             )(self, dims)
 
-        combined_rank = 0
+        combined_rank: Any = 0
         for dim in dims:
             combined_rank *= self.size(dim)
             combined_rank += self.rank(dim)
         return combined_rank
 
     @property
-    def ranks(self) -> dict[str, int]:
+    def ranks(self) -> dict[str, torch.Tensor]:
         return {dim: self.rank(dim) for dim in self.names}
 
     def process_idx(self):
@@ -334,6 +334,10 @@ class _ActiveMesh(TorchDispatchMode):
         return _remote(func, propagate=func)(*args, **kwargs)
 
 
+def _rank(mesh, dim):
+    return torch.full((), mesh.dims[dim].rank, dtype=torch.long)
+
+
 @contextmanager
 def _dispatch():
     global _dispatch_enabled
@@ -401,7 +405,7 @@ def to_mesh(
 
 def slice_mesh(
     tensors: Any,
-    **kwargs: Dict[str, Union[int, slice]],
+    **kwargs: Union[int, slice],
 ) -> Any:
     """
     Performs the slice_mesh operation for each tensor in tensors.

monarch/common/shape.py

@@ -44,6 +44,9 @@ class MeshTrait(ABC):
     @abstractmethod
     def _labels(self) -> Tuple[str, ...]: ...
 
+    # mesh trait guarentees that its own calls to _new_with_shape
+    # will only ever select a shape that is a subspace of the
+    # current _ndslice.
     @abstractmethod
     def _new_with_shape(self, shape: Shape) -> Self: ...
 

monarch/common/tensor.py

@@ -7,17 +7,20 @@
 # pyre-unsafe
 import itertools
 import traceback
+import typing
 import warnings
 from collections import defaultdict
 from typing import (
     Any,
     Callable,
+    cast,
     Dict,
     Iterable,
     List,
     Literal,
     NamedTuple,
     Optional,
+    runtime_checkable,
     Sequence,
     TYPE_CHECKING,
     TypeVar,
@@ -35,7 +38,8 @@ from .base_tensor import BaseTensor
 from .borrows import StorageAliases
 
 if TYPE_CHECKING:
-    from .device_mesh import DeviceMesh
+    from monarch.common.device_mesh import DeviceMesh
+
 from .fake import fake_call
 from .function import Propagator, ResolvableFunction
 from .invocation import Invocation
@@ -52,6 +56,12 @@ _valid_reduce = Literal[
 T = TypeVar("T")
 
 
+@runtime_checkable
+class HasDeviceMesh(typing.Protocol):
+    @property
+    def _device_mesh(self) -> "DeviceMesh": ...
+
+
 class DropLocation(NamedTuple):
     tensor_id: int
     traceback: List[traceback.FrameSummary]
@@ -167,7 +177,11 @@ class Tensor(Referenceable, BaseTensor):
             self._on_first_use(self)
             self._on_first_use = None
 
-    def to_mesh(self, mesh: "DeviceMesh", stream: Optional["Stream"] = None):
+    def to_mesh(
+        self,
+        mesh: Union["DeviceMesh", "HasDeviceMesh"],
+        stream: Optional["Stream"] = None,
+    ):
         """
         Move data between one device mesh and another. Sizes of named dimensions must match.
         If mesh has dimensions that self.mesh does not, it will broadcast to those dimensions.
@@ -177,6 +191,8 @@ class Tensor(Referenceable, BaseTensor):
             t.slice_mesh(batch=0).to_mesh(t.mesh)
 
         """
+        if isinstance(mesh, HasDeviceMesh):
+            mesh = mesh._device_mesh
         return MeshSliceTensor(self, self.mesh).to_mesh(mesh, stream)
 
     def reduce_(
@@ -344,7 +360,7 @@ class Tensor(Referenceable, BaseTensor):
         )
         return r
 
-    def slice_mesh(self, **kwargs: Dict[str, Union[int, slice]]) -> "MeshSliceTensor":
+    def slice_mesh(self, **kwargs: Union[int, slice]) -> "MeshSliceTensor":
         # technically a slice of a device mesh and a device mesh are not same thing
         # because a device mesh also has caches for doing collectives.
         # but this is an easy way to create a MeshSliceTensor until we optimize
@@ -368,8 +384,13 @@ class MeshSliceTensor:
         self.slicing = slicing
 
     def to_mesh(
-        self, mesh: "DeviceMesh", stream: Optional["Stream"] = None
+        self,
+        mesh: Union["DeviceMesh", "HasDeviceMesh"],
+        stream: Optional["Stream"] = None,
     ) -> "Tensor":
+        if isinstance(mesh, HasDeviceMesh):
+            mesh = mesh._device_mesh
+
         if stream is None:
             stream = self.tensor.stream
 

monarch/mesh_controller.py

@@ -11,7 +11,7 @@ import time
 import traceback
 from collections import deque
 from logging import Logger
-from typing import List, NamedTuple, Optional, Union
+from typing import List, NamedTuple, Optional, TYPE_CHECKING, Union
 
 import torch.utils._python_dispatch
 
@@ -24,7 +24,13 @@ from monarch._rust_bindings.monarch_extension.mesh_controller import _Controller
 from monarch._rust_bindings.monarch_hyperactor.proc import (  # @manual=//monarch/monarch_extension:monarch_extension
     ActorId,
 )
-from monarch._rust_bindings.monarch_hyperactor.proc_mesh import ProcMesh as HyProcMesh
+
+if TYPE_CHECKING:
+    from monarch._rust_bindings.monarch_hyperactor.proc_mesh import (
+        ProcMesh as HyProcMesh,
+    )
+    from monarch.proc_mesh import ProcMesh
+
 from monarch._rust_bindings.monarch_hyperactor.shape import Point
 
 from monarch._rust_bindings.monarch_messages.debugger import DebuggerAction
@@ -33,7 +39,6 @@ from monarch.common.controller_api import LogMessage, MessageResult
 from monarch.common.device_mesh import DeviceMesh, no_mesh
 from monarch.common.invocation import DeviceException, RemoteException
 from monarch.controller.debugger import read as debugger_read, write as debugger_write
-from monarch.proc_mesh import ProcMesh
 from monarch.rust_local_mesh import _get_worker_exec_info
 from pyre_extensions import none_throws
 
@@ -41,7 +46,7 @@ logger: Logger = logging.getLogger(__name__)
 
 
 class Controller(_Controller):
-    def __init__(self, workers: HyProcMesh) -> None:
+    def __init__(self, workers: "HyProcMesh") -> None:
         super().__init__()
         # Buffer for messages unrelated to debugging that are received while a
         # debugger session is active.
@@ -250,7 +255,7 @@ class MeshClient(Client):
         self.inner.drain_and_stop()
 
 
-def spawn_tensor_engine(proc_mesh: ProcMesh) -> DeviceMesh:
+def spawn_tensor_engine(proc_mesh: "ProcMesh") -> DeviceMesh:
     # This argument to Controller
     # is currently only used for debug printing. It should be fixed to
     # report the proc ID instead of the rank it currently does.

monarch/proc_mesh.py

@@ -7,8 +7,22 @@
 # pyre-strict
 
 import sys
+from contextlib import AbstractContextManager
+
+from typing import (
+    Any,
+    cast,
+    Dict,
+    List,
+    Optional,
+    Sequence,
+    Type,
+    TYPE_CHECKING,
+    TypeVar,
+)
 
-from typing import Any, cast, List, Optional, Type, TypeVar
+if TYPE_CHECKING:
+    import torch
 
 import monarch
 from monarch import ActorFuture as Future
@@ -24,7 +38,9 @@ from monarch._rust_bindings.monarch_hyperactor.shape import Shape, Slice
 from monarch.actor_mesh import _Actor, _ActorMeshRefImpl, Actor, ActorMeshRef
 
 from monarch.common._device_utils import _local_device_count
+from monarch.common.device_mesh import DeviceMesh
 from monarch.common.shape import MeshTrait
+from monarch.mesh_controller import spawn_tensor_engine
 from monarch.rdma import RDMAManager
 
 T = TypeVar("T")
@@ -45,25 +61,43 @@ def _allocate_blocking(alloc: Alloc) -> "ProcMesh":
 
 
 class ProcMesh(MeshTrait):
-    def __init__(self, hy_proc_mesh: HyProcMesh) -> None:
+    def __init__(
+        self,
+        hy_proc_mesh: HyProcMesh,
+        _mock_shape: Optional[Shape] = None,
+        _device_mesh: Optional[DeviceMesh] = None,
+    ) -> None:
         self._proc_mesh = hy_proc_mesh
+        self._mock_shape: Optional[Shape] = _mock_shape
         self._mailbox: Mailbox = self._proc_mesh.client
-        self._rdma_manager: RDMAManager = self._spawn_blocking(
-            "rdma_manager", RDMAManager
-        )
+        self._rdma_manager: Optional[RDMAManager] = None
+        self._maybe_device_mesh: Optional[DeviceMesh] = _device_mesh
+        if _mock_shape is None:
+            self._rdma_manager = self._spawn_blocking("rdma_manager", RDMAManager)
+
+    @property
+    def _shape(self) -> Shape:
+        return self._proc_mesh.shape if self._mock_shape is None else self._mock_shape
 
     @property
     def _ndslice(self) -> Slice:
-        return self._proc_mesh.shape.ndslice
+        return self._shape.ndslice
 
     @property
     def _labels(self) -> List[str]:
-        return self._proc_mesh.shape.labels
+        return self._shape.labels
 
     def _new_with_shape(self, shape: Shape) -> "ProcMesh":
-        raise NotImplementedError("ProcMesh slicing is not implemeted yet.")
+        device_mesh = (
+            None
+            if self._device_mesh is None
+            else self._device_mesh._new_with_shape(shape)
+        )
+        return ProcMesh(self._proc_mesh, _mock_shape=shape, _device_mesh=device_mesh)
 
     def spawn(self, name: str, Class: Type[T], *args: Any, **kwargs: Any) -> Future[T]:
+        if self._mock_shape is not None:
+            raise NotImplementedError("NYI: spawn on slice of a proc mesh.")
         return Future(
             lambda: self._spawn_nonblocking(name, Class, *args, **kwargs),
             lambda: self._spawn_blocking(name, Class, *args, **kwargs),
@@ -120,6 +154,26 @@ class ProcMesh(MeshTrait):
         service._create(args, kwargs)
         return cast(T, service)
 
+    @property
+    def _device_mesh(self) -> "DeviceMesh":
+        if self._maybe_device_mesh is None:
+            if self._mock_shape is not None:
+                raise NotImplementedError(
+                    "NYI: activating a proc mesh must first happen on the root proc_mesh until we fix spawning on submeshes."
+                )
+            self._maybe_device_mesh = spawn_tensor_engine(self)
+        return self._maybe_device_mesh
+
+    # pyre-ignore
+    def activate(self) -> AbstractContextManager:
+        return self._device_mesh.activate()
+
+    def rank_tensor(self, dim: str | Sequence[str]) -> "torch.Tensor":
+        return self._device_mesh.rank(dim)
+
+    def rank_tensors(self) -> Dict[str, "torch.Tensor"]:
+        return self._device_mesh.ranks
+
 
 async def local_proc_mesh_nonblocking(
     *, gpus: Optional[int] = None, hosts: int = 1

monarch/sim_mesh.py

@@ -201,9 +201,11 @@ class Bootstrap:
 
         proxy_addr = proxy_addr or f"unix!@{_random_id()}-proxy"
         self.bootstrap_addr: str = f"sim!unix!@system,{proxy_addr}"
-        self.client_listen_addr: str = f"sim!unix!@client,{proxy_addr}"
+
+        client_proxy_addr = f"unix!@{_random_id()}-proxy"
+        self.client_listen_addr: str = f"sim!unix!@client,{client_proxy_addr}"
         self.client_bootstrap_addr: str = (
-            f"sim!unix!@client,{proxy_addr},unix!@system,{proxy_addr}"
+            f"sim!unix!@client,{client_proxy_addr},unix!@system,{proxy_addr}"
         )
         bootstrap_simulator_backend(self.bootstrap_addr, proxy_addr, world_size)
 

monarch/tools/cli.py

@@ -112,7 +112,7 @@ class InfoCmd:
                 file=sys.stderr,
             )
         else:
-            json.dump(server_spec.to_json(), fp=sys.stdout)
+            json.dump(server_spec.to_json(), indent=2, fp=sys.stdout)
 
 
 class KillCmd:

monarch/tools/components/hyperactor.py

@@ -25,6 +25,7 @@ def proc_mesh(
     meshes: list[str] = _DEFAULT_MESHES,
     env: Optional[dict[str, str]] = None,
     port: int = mesh_spec.DEFAULT_REMOTE_ALLOCATOR_PORT,
+    program: str = "monarch_bootstrap",  # installed with monarch wheel (as console script)
 ) -> specs.AppDef:
     """
     Args:
@@ -33,6 +34,7 @@ def proc_mesh(
         meshes: list of mesh specs of the form "{name}:{num_hosts}:{host_type}"
         env: environment variables to be passed to the main command (e.g. ENV1=v1,ENV2=v2,ENV3=v3)
         port: the port that the remote process allocator runs on (must be reachable from the client)
+        program: path to the binary that the remote process allocator spawns on an allocation request
     """
 
     appdef = specs.AppDef(name)
@@ -41,11 +43,10 @@ def proc_mesh(
         mesh_role = specs.Role(
             name=mesh.name,
             image=image,
-            entrypoint="process_allocator",  # 'cargo install monarch_hyperactor' to get this binary
+            entrypoint="process_allocator",  # run "cargo install monarch_hyperactor" to get this binary
             args=[
-                "mesh-worker",
                 f"--port={port}",
-                "--program=monarch_bootstrap",  # installed with monarch wheel (as console script)
+                f"--program={program}",
             ],
             num_replicas=mesh.num_hosts,
             resource=specs.resource(h=mesh.host_type),

monarch/tools/network.py

@@ -0,0 +1,62 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# pyre-strict
+import logging
+import socket
+from typing import Optional
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+def get_ip_addr(hostname: str) -> str:
+    """Resolves and returns the ip address of the given hostname.
+
+    This function will return an ipv6 address if one that can bind
+    `SOCK_STREAM` (TCP) socket is found. Otherwise it will fall-back
+    to resolving an ipv4 `SOCK_STREAM` address.
+
+    Raises a `RuntimeError` if neither ipv6 or ipv4 ip can be resolved from hostname.
+    """
+
+    def get_sockaddr(family: socket.AddressFamily) -> Optional[str]:
+        try:
+            # patternlint-disable-next-line python-dns-deps (only used for oss)
+            addrs = socket.getaddrinfo(
+                hostname, port=None, family=family, type=socket.SOCK_STREAM
+            )  # tcp
+            if addrs:
+                # socket.getaddrinfo return a list of addr 5-tuple addr infos
+                _, _, _, _, sockaddr = addrs[0]  # use the first address
+
+                # sockaddr is a tuple (ipv4) or a 4-tuple (ipv6) where the first element is the ip addr
+                ipaddr = str(sockaddr[0])
+
+                logger.info(
+                    "Resolved %s address: `%s` for host: `%s`",
+                    family.name,
+                    ipaddr,
+                    hostname,
+                )
+                return str(ipaddr)
+            else:
+                return None
+        except socket.gaierror as e:
+            logger.info(
+                "No %s address that can bind TCP sockets for host: %s. %s",
+                family.name,
+                hostname,
+                e,
+            )
+            return None
+
+    ipaddr = get_sockaddr(socket.AF_INET6) or get_sockaddr(socket.AF_INET)
+    if not ipaddr:
+        raise RuntimeError(
+            f"Unable to resolve `{hostname}` to ipv6 or ipv4 address that can bind TCP socket."
+            " Check the network configuration on the host."
+        )
+    return ipaddr

tests/error_test_binary.py

@@ -4,6 +4,7 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import asyncio
 import ctypes
 import sys
 
@@ -11,7 +12,7 @@ import click
 
 from monarch._rust_bindings.monarch_extension.panic import panicking_function
 
-from monarch.actor_mesh import Actor, endpoint
+from monarch.actor_mesh import Actor, endpoint, send
 from monarch.proc_mesh import proc_mesh
 
 
@@ -35,6 +36,12 @@ class ErrorActor(Actor):
         """Endpoint that calls a Rust function that panics."""
         panicking_function()
 
+    @endpoint
+    async def await_then_error(self) -> None:
+        await asyncio.sleep(0.1)
+        await asyncio.sleep(0.1)
+        raise RuntimeError("oh noez")
+
 
 class ErrorActorSync(Actor):
     """An actor that has endpoints cause segfaults."""
@@ -146,5 +153,28 @@ def error_bootstrap():
     proc_mesh(gpus=4, env={"MONARCH_ERROR_DURING_BOOTSTRAP_FOR_TESTING": "1"}).get()
 
 
+async def _error_unmonitored():
+    print("I actually ran")
+    sys.stdout.flush()
+
+    proc = await proc_mesh(gpus=1)
+    actor = await proc.spawn("error_actor", ErrorActor)
+
+    # fire and forget
+    send(actor.await_then_error, (), {}, None, "all")
+
+    # Wait. Eventually a supervision event will get propagated and the process
+    # will exit.
+    #
+    # If an event is not delivered, the test will time out before this sleep
+    # finishes.
+    await asyncio.sleep(300)
+
+
+@main.command("error-unmonitored")
+def error_unmonitored():
+    asyncio.run(_error_unmonitored())
+
+
 if __name__ == "__main__":
     main()

tests/test_actor_error.py

@@ -4,11 +4,12 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import asyncio
 import importlib.resources
 import subprocess
 
 import pytest
-from monarch.actor_mesh import Actor, ActorError, endpoint
+from monarch.actor_mesh import Actor, ActorError, endpoint, send
 
 from monarch.proc_mesh import proc_mesh
 
@@ -128,6 +129,7 @@ def test_actor_supervision(num_procs, sync_endpoint, sync_test_impl, endpoint_na
         f"--endpoint-name={endpoint_name}",
     ]
     try:
+        print("running cmd", " ".join(cmd))
         process = subprocess.run(cmd, capture_output=True, timeout=180)
     except subprocess.TimeoutExpired as e:
         print("timeout expired")
@@ -157,6 +159,7 @@ def test_proc_mesh_bootstrap_error():
         "error-bootstrap",
     ]
     try:
+        print("running cmd", " ".join(cmd))
         process = subprocess.run(cmd, capture_output=True, timeout=180)
     except subprocess.TimeoutExpired as e:
         print("timeout expired")
@@ -208,3 +211,30 @@ async def test_broken_pickle_class(raise_on_getstate, raise_on_setstate, num_pro
             await exception_actor.print_value.call_one(broken_obj)
         else:
             await exception_actor.print_value.call(broken_obj)
+
+
+# oss_skip: importlib not pulling resource correctly in git CI, needs to be revisited
+@pytest.mark.oss_skip
+async def test_exception_after_wait_unmonitored():
+    # Run the test in a subprocess
+    test_bin = importlib.resources.files("monarch.python.tests").joinpath("test_bin")
+    cmd = [
+        str(test_bin),
+        "error-unmonitored",
+    ]
+    try:
+        print("running cmd", " ".join(cmd))
+        process = subprocess.run(cmd, capture_output=True, timeout=180)
+    except subprocess.TimeoutExpired as e:
+        print("timeout expired")
+        if e.stdout is not None:
+            print(e.stdout.decode())
+        if e.stderr is not None:
+            print(e.stderr.decode())
+        raise
+
+    # Assert that the subprocess exited with a non-zero code
+    assert "I actually ran" in process.stdout.decode()
+    assert (
+        process.returncode != 0
+    ), f"Expected non-zero exit code, got {process.returncode}"

tests/test_python_actors.py

@@ -391,10 +391,13 @@ def test_rust_binding_modules_correct() -> None:
     check(bindings, "monarch._rust_bindings")
 
 
-@pytest.mark.skipif(
+two_gpu = pytest.mark.skipif(
     torch.cuda.device_count() < 2,
     reason="Not enough GPUs, this test requires at least 2 GPUs",
 )
+
+
+@two_gpu
 def test_tensor_engine() -> None:
     pm = proc_mesh(gpus=2).get()
 
@@ -581,13 +584,80 @@ async def test_actor_tls() -> None:
     pm = await proc_mesh(gpus=1)
     am = await pm.spawn("tls", TLSActor)
     await am.increment.call_one()
-    # TODO(suo): TLS is NOT preserved across async/sync endpoints, because currently
-    # we run async endpoints on a different thread than sync ones.
-    # Will fix this in a followup diff.
+    await am.increment_async.call_one()
+    await am.increment.call_one()
+    await am.increment_async.call_one()
+
+    assert 4 == await am.get.call_one()
+    assert 4 == await am.get_async.call_one()
+
+
+class TLSActorFullSync(Actor):
+    """An actor that manages thread-local state."""
+
+    def __init__(self):
+        self.local = threading.local()
+        self.local.value = 0
+
+    @endpoint
+    def increment(self):
+        self.local.value += 1
+
+    @endpoint
+    def get(self):
+        return self.local.value
+
 
-    # await am.increment_async.call_one()
+async def test_actor_tls_full_sync() -> None:
+    """Test that thread-local state is respected."""
+    pm = await proc_mesh(gpus=1)
+    am = await pm.spawn("tls", TLSActorFullSync)
+    await am.increment.call_one()
+    await am.increment.call_one()
+    await am.increment.call_one()
     await am.increment.call_one()
-    # await am.increment_async.call_one()
 
-    assert 2 == await am.get.call_one()
-    # assert 4 == await am.get_async.call_one()
+    assert 4 == await am.get.call_one()
+
+
+@two_gpu
+def test_proc_mesh_tensor_engine() -> None:
+    pm = proc_mesh(gpus=2).get()
+    with pm.activate():
+        f = 10 * pm.rank_tensor("gpus").cuda()
+        a = monarch.inspect(f, hosts=0, gpus=0)
+        b = monarch.inspect(f, hosts=0, gpus=1)
+
+    one = pm.slice(gpus=1)
+    with one.activate():
+        sliced_b = monarch.slice_mesh(f, gpus=1).to_mesh(one)
+        c = monarch.inspect(sliced_b * 10)
+    assert a == 0
+    assert b == 10
+    assert c == 100
+
+
+class AsyncActor(Actor):
+    def __init__(self):
+        self.should_exit = False
+
+    @endpoint
+    async def sleep(self) -> None:
+        while True and not self.should_exit:
+            await asyncio.sleep(1)
+
+    @endpoint
+    async def no_more(self) -> None:
+        self.should_exit = True
+
+
+@pytest.mark.timeout(15)
+async def test_async_concurrency():
+    """Test that async endpoints will be processed concurrently."""
+    pm = await proc_mesh(gpus=1)
+    am = await pm.spawn("async", AsyncActor)
+    fut = am.sleep.call()
+    # This call should go through and exit the sleep loop, as long as we are
+    # actually concurrently processing messages.
+    await am.no_more.call()
+    await fut

{torchmonarch_nightly-2025.6.16.dist-info → torchmonarch_nightly-2025.6.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: torchmonarch-nightly
-Version: 2025.6.16
+Version: 2025.6.18
 Summary: Monarch: Single controller library
 Author: Meta
 Author-email: oncall+monarch@xmail.facebook.com

{torchmonarch_nightly-2025.6.16.dist-info → torchmonarch_nightly-2025.6.18.dist-info}/RECORD

@@ -1,7 +1,7 @@
 monarch/__init__.py,sha256=iUvWHc0-7Q2tovRoRxOIiA3TsefMXCbWl-jEfQ2djew,6897
-monarch/_rust_bindings.so,sha256=Mus3Wdk7VoEHkyPLV1_SQ2r2KAMOPNTv3rECuKH5Olk,40613688
+monarch/_rust_bindings.so,sha256=RlkNuWQ74oxTOEfmaVFsgESTEdMP84vug1sRY4xya60,40803008
 monarch/_testing.py,sha256=jOIOG6jcZBzvEvG_DwSnwCkaMVXvSun6sJAG6nXemww,7859
-monarch/actor_mesh.py,sha256=nAW65WFEWMJWCv8zuH9GSOyTNXwFN8QNqZxMZTuSYxw,25537
+monarch/actor_mesh.py,sha256=8hjIy0TSby33xfVXp_xZnqlPkxy3l6IGqEyPOhVtjvU,24197
 monarch/allocator.py,sha256=ylvYTf31o-PT385cYJPhi17uNbC4yl_RAraqD0fVe4g,4112
 monarch/bootstrap_main.py,sha256=RCUQhJk07yMFiKp6HzQuqZFUpkgsT9kVEyimiwjn6_E,1827
 monarch/cached_remote_function.py,sha256=kYdB6r4OHx_T_uX4q3tCNcp1t2DJwF8tPTIahUiT2pU,8785
@@ -10,13 +10,13 @@ monarch/fetch.py,sha256=61jxo7sx4QNUTkc0_rF5NaJROen4tKbAaiIjrXWLOvg,1705
 monarch/future.py,sha256=lcdFEe7m1shYPPuvZ1RkS6JUIChEKGBWe3v7x_nu4Hg,731
 monarch/gradient_generator.py,sha256=Rl3dmXGceTdCc1mYBg2JciR88ywGPnW7TVkL86KwqEA,6366
 monarch/memory.py,sha256=ol86dBhFAJqg78iF25-BuK0wuwj1onR8FIioZ_B0gjw,1377
-monarch/mesh_controller.py,sha256=Xft2edk7rz8_PPe-iIUZ09P-j4JDPGADBGHBiuiZ7YY,10363
-monarch/monarch_controller,sha256=OZYuYEUToULTaOxmUk3Dv-73n68gFua8z4pP5WCwU5I,20400832
+monarch/mesh_controller.py,sha256=am1QP7dvn0OH1z9ADSKm41APs1HY_dHcBAhOVP-QDmE,10427
+monarch/monarch_controller,sha256=HucZG4CSJhkVpbHElarAp2LUz1xW5bMNnAR3TNjWKks,20335344
 monarch/notebook.py,sha256=zu9MKDFKf1-rCM2TqFSRJjMBeiWuKcJSyUFLvoZRQzs,25949
 monarch/opaque_module.py,sha256=oajOu_WD1hD4hxE8HDdO-tvWY7KDHWd7VaAhJEa5L2I,10446
 monarch/opaque_object.py,sha256=IVpll4pyuKZMo_EnPh4s0qnx8RlAcJrJ1yoLX6E75wQ,2782
 monarch/pdb_wrapper.py,sha256=gm46AZnfR4amH1vYFWnWivEv5MaU3Nb6KIWjSM8KjWM,4052
-monarch/proc_mesh.py,sha256=xoaReM9Ab9TWkesxedWSyyk4TMD0HLV88dQ8CQcbqTI,6892
+monarch/proc_mesh.py,sha256=5RaKPQZJD-sKzEAbqMorKsZA7SOUzIflk3Fn6cdfzvw,8607
 monarch/profiler.py,sha256=TQ9fnVM8H7smBWtYdB_6Irtzz8DBOmcp7U1T3wlUmco,4911
 monarch/python_local_mesh.py,sha256=YsureIzR9uGlNVrKd4vRghxOXBeYabkt9lICRErfRAI,3536
 monarch/random.py,sha256=f9QR7Esu4Vxqxs-KCf5QYyVqlWvXJ3-UtG90L_h4j40,1527
@@ -24,7 +24,7 @@ monarch/rdma.py,sha256=1pNh11S_FWeETRgkdUpauTMUlodrRohIq1UfQjKVnN8,5418
 monarch/remote_class.py,sha256=-OAowzU1aDP6i4ik_SjXntVUC9h4dqAzgqwohkQ6Grc,4167
 monarch/rust_backend_mesh.py,sha256=1htC62of4MgFtkezWGlsxSFtKJdc0CIeqeSuOx7yu3M,9944
 monarch/rust_local_mesh.py,sha256=7ASptybn3wy4J7eoBc7LhGW4j4AA6bigl5Kuhyflw8s,47405
-monarch/sim_mesh.py,sha256=9wkS99L0EpG2Gldi-nzA-3ww7z__DQ7Qp2uReMfn188,12183
+monarch/sim_mesh.py,sha256=kDsbubv28YFg9ZQN4urt3oJGzR3CnnUiATnjUiSxrkE,12256
 monarch/telemetry.py,sha256=7JUZWaoD2Yn5Ae_7kNhkLFRBLYaSGfH071_m_qfVehI,525
 monarch/tensor_worker_main.py,sha256=Nbarl2sJKIddLeaRFsaUnqOerLHjzggUr9SqCr2_GYI,8300
 monarch/tensorboard.py,sha256=MnLgH5lbqeUJauEuirEgR6L_qYl2NGdtwZOWIAuOZao,2587
@@ -50,7 +50,7 @@ monarch/common/client.py,sha256=axo37s_z17nYQGOZG5fi_0zUEJ_8qw7INjs-Kw2vaVo,2493
 monarch/common/constants.py,sha256=ohvsVYMpfeWopv3KXDAeHWDFLukwc-OY37VRxpKNBE8,300
 monarch/common/context_manager.py,sha256=GOeyaFbyCqvQmkJ0oI7q6IxRd8_0mVyYKZRccI8iaug,1067
 monarch/common/controller_api.py,sha256=djGkK5aSd-V6pBkr3uBCXbfJv3OKf2o2VbBXJgFF2WI,3202
-monarch/common/device_mesh.py,sha256=fBZMYDpfAp5tAEXTe9l6eJxDI4-TMWVOMrAJXp5hzvI,12082
+monarch/common/device_mesh.py,sha256=jo_qEIRlX6KzBlP2BUSS4XEELL-6_H08a47bUz8QYsA,12159
 monarch/common/fake.py,sha256=h57Cggz2qXNqImZ7yPuOZOSe9-l9i553ki1z-YHlgQA,1801
 monarch/common/function.py,sha256=V8kdgSRTvild2SpcewWa5IETX3QiWDZQ2BEIDFa5zz8,4374
 monarch/common/function_caching.py,sha256=HVdbWtv6Eea7ENMWi8iv36w1G1TaVuUJhkUX_JxGx5A,5060
@@ -67,9 +67,9 @@ monarch/common/recording.py,sha256=hoI9VY_FyW_xVx-jmfsKydqX5vW2GulwcDWsBdUVOm8,4
 monarch/common/reference.py,sha256=O26lkzEeVwj0S1xEy-OLqdHVnACmmlbQCUmXRrW4n1Q,938
 monarch/common/remote.py,sha256=qZWXkShX20l07TseQSpVECh2yXZaVKYUvQXkeEM-zvY,9220
 monarch/common/selection.py,sha256=lpWFbZs3ArYy29e-53eoAVAjQFksf1RvZz9NvM0CUW4,308
-monarch/common/shape.py,sha256=k6-0S0U19PmrfP62SMb9Ihx6_I4QQFUGErloZn8GcZ0,8144
+monarch/common/shape.py,sha256=B-7DI768ZhT8ECUNCJcI7DfCB7iDFGFH0r-HmXaAfcM,8296
 monarch/common/stream.py,sha256=_ejoxafHtdD10lLzznRCXKwrkZ_ZH9k_VTgiA5yfBrI,3583
-monarch/common/tensor.py,sha256=mSXiHoD0Up4m2RLdQcsbesaz2N4QCFS34UNNX3Dbldk,28842
+monarch/common/tensor.py,sha256=G26E8-qv7HnjZfz3Ka5a-u3vb6DadcDChOn6wpjkeZo,29273
 monarch/common/tensor_factory.py,sha256=qm8NZx-5ezMAFjNLiXQvb66okm5XgdboB_GRarGOdN0,801
 monarch/common/tree.py,sha256=1DG3siiE7ixBV6v5cwN8RT_17aJhYZTE-L3i7wZe2_c,2282
 monarch/controller/__init__.py,sha256=J8qjUOysmcMAek2KFN13mViOXZxTYc5vCrF02t3VuFU,223
@@ -106,11 +106,12 @@ monarch/timer/example_spmd.py,sha256=p8i3_tO1AmpwSkZryiSjgkh7qaEZ6QXp2Fy1qtPpECA
 monarch/timer/execution_timer.py,sha256=1YsrLIZirdohKOeFAU2H4UcONhQXHuctJbYcoX8I6gY,6985
 monarch/timer/execution_timer_test.py,sha256=CSxTv44fFZQURJlCBmYvysQI1aS_zEGZs_uxl9SOHak,4486
 monarch/tools/__init__.py,sha256=J8qjUOysmcMAek2KFN13mViOXZxTYc5vCrF02t3VuFU,223
-monarch/tools/cli.py,sha256=66F7dr90bh27P3kOCmxwJkVmWv2v4wBrkifvwqwUwFE,4967
+monarch/tools/cli.py,sha256=EIdarsfuFX0WqRCe29_5GNKWJBhxx0lABalw3zPSagw,4977
 monarch/tools/commands.py,sha256=BfmXndJmU_cZP4cMPlknkxGca1NjqYd8_ReDePWksXw,6908
 monarch/tools/mesh_spec.py,sha256=JLykhgy1dClXiNbH1Qsl2fX5MbqplQAhl8LGoragvbo,3702
+monarch/tools/network.py,sha256=bRj-jOs5qDqnM3BcE9MSXCLS01hiMN4YSWfKZ_d7bc4,2182
 monarch/tools/components/__init__.py,sha256=J8qjUOysmcMAek2KFN13mViOXZxTYc5vCrF02t3VuFU,223
-monarch/tools/components/hyperactor.py,sha256=h0gy3QYZD-YJ7FHppJgbTKe4zOuNjUCGZqRlkwwGkhg,2012
+monarch/tools/components/hyperactor.py,sha256=Ryi1X07VLcaQVlpc4af65JNBbZtOb9IAlKxSKMZ1AW4,2120
 monarch/tools/config/__init__.py,sha256=OPSflEmJB2zxAaRVzzWSWXV5M5vlknLgpulGdW1ze5U,510
 monarch/tools/config/defaults.py,sha256=34a3HQhyXqt9qR2SYMVCROoNsnwk37rIwLXXiKwqtog,1894
 monarch/worker/__init__.py,sha256=J8qjUOysmcMAek2KFN13mViOXZxTYc5vCrF02t3VuFU,223
@@ -131,9 +132,9 @@ monarch_supervisor/python_executable.py,sha256=WfCiK3wdAvm9Jxx5jgjGF991NgGc9-oHU
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/dispatch_bench.py,sha256=sU_m-8KAjQgYTsxI5khV664NdgLLutidni69Rtowk98,3933
 tests/dispatch_bench_helper.py,sha256=1ORgAMrRgjAjmmWeCHLLQd_bda9mJk0rS2ucEbRu28s,633
-tests/error_test_binary.py,sha256=64H-ucdkQ2i7GD8sidStl227cOy7gyeqvO4kTm1y7Ic,4817
+tests/error_test_binary.py,sha256=BRj13wAROsUWx4jcxc07HYN2n-xyBNhnnRAhjqah-A0,5582
 tests/sleep_binary.py,sha256=XfLYaAfwm9xgzM-svs8fhAeFhwYIg6SyVEnx4e6wbUw,1009
-tests/test_actor_error.py,sha256=z3Sf4lteUggTryPLOhRKJ55v0MwVK3a7QN7-U2U9iJg,7484
+tests/test_actor_error.py,sha256=-0UJCEpyzsBh-RdbGhDiG1-sRtu7bJPQWmtjUD0ad48,8526
 tests/test_alloc.py,sha256=D6DdQbtOZEvvnnc7LV-WyWFMk0Xb77eblH6Oz90zJTA,745
 tests/test_allocator.py,sha256=P11sQ95ADjzC_-CfPs3CEP80nP8sn7wW8vVPsmpSVoM,8164
 tests/test_coalescing.py,sha256=JZ4YgQNlWWs7N-Z8KCCXQPANcuyyXEKjeHIXYbPnQhk,15606
@@ -144,7 +145,7 @@ tests/test_future.py,sha256=cXzaNi2YDwVyjR541ScXmgktX1YFsKzbl8wep0DMVbk,3032
 tests/test_grad_generator.py,sha256=p4Pm4kMEeGldt2jUVAkGKCB0mLccKI28pltH6OTGbQA,3412
 tests/test_mock_cuda.py,sha256=5hisElxeLJ5MHw3KM9gwxBiXiMaG-Rm382u3AsQcDOI,3068
 tests/test_pdb_actor.py,sha256=5KJhuhcZDPWMdjC6eAtDdwnz1W7jNFXvIrMSFaCWaPw,3858
-tests/test_python_actors.py,sha256=MzGeuhGVICzwiNaQt8SFCKyfwhNzdRzZ4s2rJxYbeoo,17283
+tests/test_python_actors.py,sha256=ls0x_ie4i9KLuouecfxG_fHHZSZc2g_mQSAPJg70pgw,18949
 tests/test_remote_functions.py,sha256=5nxYB8dfA9NT9f9Od9O3htgQtPbiRNiXZ1Kgtn75sOQ,50056
 tests/test_rust_backend.py,sha256=94S3R995ZkyIhEiBsM5flcjf5X7bscEAHBtInbTRFe8,7776
 tests/test_signal_safe_block_on.py,sha256=bmal0XgzJowZXJV6T1Blow5a-vZluYWusCThLMGxyTE,3336
@@ -154,9 +155,9 @@ tests/simulator/test_profiling.py,sha256=TGYCfzTLdkpIwnOuO6KApprmrgPIRQe60KRX3wk
 tests/simulator/test_simulator.py,sha256=LO8lA0ssY-OGEBL5ipEu74f97Y765TEwfUOv-DtIptM,14568
 tests/simulator/test_task.py,sha256=ipqBDuDAysuo1xOB9S5psaFvwe6VATD43IovCTSs0t4,2327
 tests/simulator/test_worker.py,sha256=QrWWIJ3HDgDLkBPRc2mwYPlOQoXQcj1qRfc0WUfKkFY,3507
-torchmonarch_nightly-2025.6.16.dist-info/licenses/LICENSE,sha256=e0Eotbf_rHOYPuEUlppIbvwy4SN98CZnl_hqwvbDA4Q,1530
-torchmonarch_nightly-2025.6.16.dist-info/METADATA,sha256=KwDmmYW1hUyjyax5yF9TE1Tk7JvYmylSpIe4e2T_aXI,2772
-torchmonarch_nightly-2025.6.16.dist-info/WHEEL,sha256=_wZSFk0d90K9wOBp8Q-UGxshyiJ987JoPiyUBNC6VLk,104
-torchmonarch_nightly-2025.6.16.dist-info/entry_points.txt,sha256=sqfQ16oZqjEvttUI-uj9BBXIIE6jt05bYFSmy-2hyXI,106
-torchmonarch_nightly-2025.6.16.dist-info/top_level.txt,sha256=E-ZssZzyM17glpVrh-S9--qJ-w9p2EjuYOuNw9tQ4Eg,33
-torchmonarch_nightly-2025.6.16.dist-info/RECORD,,
+torchmonarch_nightly-2025.6.18.dist-info/licenses/LICENSE,sha256=e0Eotbf_rHOYPuEUlppIbvwy4SN98CZnl_hqwvbDA4Q,1530
+torchmonarch_nightly-2025.6.18.dist-info/METADATA,sha256=lPDac3GQrS5MmEp41wt6YCWHIluJzBgFfPY37x0cKJM,2772
+torchmonarch_nightly-2025.6.18.dist-info/WHEEL,sha256=_wZSFk0d90K9wOBp8Q-UGxshyiJ987JoPiyUBNC6VLk,104
+torchmonarch_nightly-2025.6.18.dist-info/entry_points.txt,sha256=sqfQ16oZqjEvttUI-uj9BBXIIE6jt05bYFSmy-2hyXI,106
+torchmonarch_nightly-2025.6.18.dist-info/top_level.txt,sha256=E-ZssZzyM17glpVrh-S9--qJ-w9p2EjuYOuNw9tQ4Eg,33
+torchmonarch_nightly-2025.6.18.dist-info/RECORD,,