torchx-nightly 2023.3.21__py3-none-any.whl → 2023.3.22__py3-none-any.whl

torchx/components/dist.py

@@ -62,6 +62,7 @@ from typing import Dict, Iterable, List, Optional, Tuple
 
 import torchx
 import torchx.specs as specs
+from torchx.components.structured_arg import StructuredJArgument, StructuredNameArgument
 from torchx.specs import macros
 
 _TORCH_DEBUG_FLAGS: Dict[str, str] = {
@@ -80,12 +81,88 @@ These are commonly set environment variables to debug PyTorch execution.
 """
 
 
+def spmd(
+    *args: str,
+    script: Optional[str] = None,
+    m: Optional[str] = None,
+    image: str = torchx.IMAGE,
+    name: str = "/",
+    h: str = "gpu.small",
+    j: str = "1x1",
+    env: Optional[Dict[str, str]] = None,
+    max_retries: int = 0,
+    mounts: Optional[List[str]] = None,
+    debug: bool = False,
+) -> specs.AppDef:
+    """
+    Usage (by script): torchx run spmd -j 2x8 -h aws_p4d.24xlarge --name my_experiment/trial_1 --script path/to/my/trainer.py -foo bar
+
+    Usage (by module): torchx run spmd -j 2x8 -h aws_p4d.24xlarge --name my_experiment/trial_1 -m path.to.my.trainer -foo bar
+
+    Usage (infer GPU count): torchx run spmd -j 2 -h p4d.24xlarge ... (same as -j 2x8)
+
+    Creates a torchx.specs.AppDef (Job Definition) for a Single-Process-Multiple-Data (SPMD)
+    style application. See: https://en.wikipedia.org/wiki/Single_program,_multiple_data.
+
+    SPMD launches `n x m` (set via the `-j nxm` option) copies of the same program,
+    where `n` is the number of nodes (hosts) and `m` is the number of processes on each node.
+
+    If you have a distributed PyTorch script (DDP, FSDP, RPC) use this component to launch
+    the distributed application. You can also use `-j 1x1` to launch a single process application
+    which would be equivalent to launching with regular `python` except that your application
+    can safely call `torch.distributed.init_process_group(backend)`.
+
+    Note: For multi-node distributed runs, the hosts MUST have a network route to each other
+          AND port 29500 should be open on all hosts. Please check your security group settings.
+
+
+    Args:
+        args: the arguments to the main module or script (e.g. my/trainer.py -foo bar)
+            (for docker based runs) the script path must be relative to the WORKDIR of the image
+        script:
+        m: the main module name (e.g. my.module.trainer). When this option is used, the `script_args` are passed
+           as the arguments to the main module). Invoking my module is useful when the relative/absolute path
+           of the main script is unknown w.r.t the WORKDIR of the image. Use this option when it makes sense to
+           invoke the main script via `python -m <MAIN.MODULE>`.
+        image: the base docker image of the workspace, if workspace is disabled, then the image of the job
+        name: ``{experimentname}/{runname}`` or ``{experimentname}/`` or ``/{runname}`` or ``{runname}``
+        h: the type of host to run on (e.g. aws_p4d.24xlarge). Must be one of the registered named resources
+        j: {nnodes}x{nproc_per_node}. For GPU hosts omitting nproc_per_node will infer it from the GPU count on the host
+        env: environment variables to be passed to the run (e.g. ENV1=v1,ENV2=v2,ENV3=v3)
+        max_retries: the number of scheduler retries allowed
+        rdzv_port: the port on rank0's host to use for hosting the c10d store used for rendezvous.
+                   Only takes effect when running multi-node. When running single node, this parameter
+                   is ignored and a random free port is chosen.
+        mounts: (for docker based runs only) mounts to mount into the worker environment/container
+                (ex. type=<bind/volume>,src=/host,dst=/job[,readonly]).
+        debug: whether to run with preset debug flags enabled
+
+    """
+
+    if env is None:
+        env = {}
+
+    return ddp(
+        *args,
+        script=script,
+        m=m,
+        image=image,
+        name=name,
+        h=h,
+        j=str(StructuredJArgument.parse_from(h, j)),
+        env=env,
+        max_retries=max_retries,
+        mounts=mounts,
+        debug=debug,
+    )
+
+
 def ddp(
     *script_args: str,
     script: Optional[str] = None,
     m: Optional[str] = None,
     image: str = torchx.IMAGE,
-    name: Optional[str] = None,
+    name: str = "/",
     h: Optional[str] = None,
     cpu: int = 2,
     gpu: int = 0,
@@ -114,7 +191,8 @@ def ddp(
         script: script or binary to run within the image
         m: the python module path to run
         image: image (e.g. docker)
-        name: job name override (uses the script name if not specified)
+        name: job name override in the following format: ``{experimentname}/{runname}`` or ``{experimentname}/`` or ``/{runname}`` or ``{runname}``.
+            Uses the script or module name if ``{runname}`` not specified.
         cpu: number of cpus per replica
         gpu: number of gpus per replica
         memMB: cpu memory in MB per replica
@@ -138,14 +216,6 @@ def ddp(
     # nproc_per_node: number of processes on each node
     min_nnodes, max_nnodes, nproc_per_node, nnodes_rep = parse_nnodes(j)
 
-    if script:
-        # script name/module no extension
-        role_name = Path(script).stem
-    elif m:
-        role_name = m.rpartition(".")[2]
-    else:
-        raise ValueError("failed to compute role_name")
-
     rdzv_backend = "c10d"
     if max_nnodes == 1:
         # using port 0 makes elastic chose a free random port which is ok
@@ -165,8 +235,16 @@ def ddp(
 
     if env is None:
         env = {}
-    env.setdefault("LOGLEVEL", os.getenv("LOGLEVEL", "WARNING"))
 
+    argname = StructuredNameArgument.parse_from(
+        name=name,
+        m=m,
+        script=script,
+    )
+
+    env["TORCHX_TRACKING_EXPERIMENT_NAME"] = argname.experiment_name
+
+    env.setdefault("LOGLEVEL", os.getenv("LOGLEVEL", "WARNING"))
     if debug:
         env.update(_TORCH_DEBUG_FLAGS)
 
@@ -193,10 +271,10 @@ def ddp(
         cmd += ["-m", m]
     cmd += script_args
     return specs.AppDef(
-        name=name or role_name,
+        name=argname.run_name,
         roles=[
             specs.Role(
-                name=role_name,
+                name=get_role_name(script, m),
                 image=image,
                 min_replicas=min_nnodes,
                 entrypoint="bash",
@@ -214,6 +292,17 @@ def ddp(
     )
 
 
+def get_role_name(script: Optional[str], m: Optional[str]) -> str:
+    if script:
+        # script name/module no extension
+        role_name = Path(script).stem
+    elif m:
+        role_name = m.rpartition(".")[2]
+    else:
+        raise ValueError("failed to compute role_name")
+    return role_name
+
+
 def _args_join(args: Iterable[str]) -> str:
     """
     _args_join is like shlex.join but if the argument is wrapped in _noquote

torchx/components/structured_arg.py

@@ -0,0 +1,236 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Defines methods for structured (higher order) component argument parsing.
+Use the functionalities defined in this module to author components
+in such a way that the structured component arguments are consistent across the board.
+
+A structured component argument is a function argument to a component (a function that returns an ``AppDef``)
+that is human-friendly (less typing in the CLI or better readability) but technically embeds multiple
+primitive arguments.
+
+Examples:
+
+#. ``-j {NNODES}x{NPROC_PER_NODE}`` (e.g. ``-j 1x2``): Compactly represents the number of
+    nodes and number of workers per node for a distributed application. Otherwise would've had to be taken
+    as two separate arguments: ``--nnodes 1 --nproc_per_node 8``
+#. ``--name {EXPERIMENT_NAME}/{RUN_NAME}`` (e.g. ``--name t5_modeling/bfloat16_trial``): Uses a single
+    ``--name`` parameter to parse experiment and run names for logging experiments and trials (runs)
+    with an experiment tracker. The ``/`` delimiter is a natural way to group runs within experiments.
+
+"""
+import warnings
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+from pyre_extensions import none_throws
+
+from torchx import specs
+
+
+@dataclass
+class StructuredNameArgument:
+    experiment_name: str
+    run_name: str
+
+    def __str__(self) -> str:
+        return f"{self.experiment_name or ''}/{self.run_name}"
+
+    @staticmethod
+    def parse_from(
+        name: str,
+        m: Optional[str] = None,
+        script: Optional[str] = None,
+        default_experiment_name: str = "default-experiment",
+    ) -> "StructuredNameArgument":
+        """
+        Creates an :py:class:`ArgName` from the component arguments:
+        ``name``, ``m`` (main module), ``script`` (main script).
+
+        The ``name`` MUST be of the form ``{EXPERIMENT_NAME}/{RUN_NAME}`` where either or both
+        ``{EXPERIMENT_NAME}`` and ``{RUN_NAME}`` may be left empy.
+        However, the ``name`` must include the ``/`` delimiter.
+
+        For instance:
+
+        #. ``foo/``: specifies an experiment name but no run name
+        #. ``/bar``: specifies a run name but no experiment name
+        #. ``foo``: specifieds a run name but no experiment name
+        #. ``foo/bar``: specifies both experiment and run names
+        #. ``/``: does not specify experiment nor run name
+
+        If the run name is left empty then one is derived from either the ``m``
+        or ``script``, whichever is not null.
+
+        .. important::
+            Exactly one of ``m`` (main module)` or ``script`` (path to script) must be provided.
+            If both or neither are provided, then this function throws.
+
+
+        Examples:
+            .. doctest::
+
+                >>> import os
+                >>> from torchx.components.structured_arg import StructuredNameArgument
+                >>> StructuredNameArgument.parse_from(name="foo/bar", script="bar/baz.py")
+                StructuredNameArgument(experiment_name='foo', run_name='bar')
+
+                >>> StructuredNameArgument.parse_from(name="foo/", script="bar/baz.py")
+                StructuredNameArgument(experiment_name='foo', run_name='baz')
+
+                >>> StructuredNameArgument.parse_from(name="/bar", script="bar/baz.py")
+                StructuredNameArgument(experiment_name='default-experiment', run_name='bar')
+
+                >>> StructuredNameArgument.parse_from(name="foobar", m="foo.bar")
+                StructuredNameArgument(experiment_name='default-experiment', run_name='foobar')
+
+                >>> StructuredNameArgument.parse_from(name="foo/bar", m="foo.bar.baz")
+                StructuredNameArgument(experiment_name='foo', run_name='bar')
+
+                >>> StructuredNameArgument.parse_from(name="foo/", m="foo.bar.baz")
+                StructuredNameArgument(experiment_name='foo', run_name='baz')
+
+                >>> StructuredNameArgument.parse_from(name="/bar", m="foo.bar.baz")
+                StructuredNameArgument(experiment_name='default-experiment', run_name='bar')
+
+                >>> StructuredNameArgument.parse_from(name="foo/bar")
+                Traceback (most recent call last):
+                    ...
+                ValueError: No main module or script specified.
+
+
+        Arguments:
+            name: ``{EXPERIMENT_NAME}/{RUN_NAME}``, ``/{RUN_NAME}``, or ``{EXPERIMENT_NAME}/``, or ``{RUN_NAME}``
+            m: the main module (e.g. ``foo.bar.baz`` for ``foo/bar/baz.py``)
+            script: path to the main script
+
+
+        Raises:
+            ValueError: if both ``m`` and ``script`` are empty or both are non-empty.
+            ValueError: if the ``name`` does not contain the experiment/run name delimiter: ``/``.
+        """
+        if not m and not script:
+            raise ValueError(
+                "No main module or script specified. Specify either a main module or a script path"
+            )
+        if m and script:
+            raise ValueError(
+                "Both main module and script set. Specify exactly one of: main module or script, but not both"
+            )
+
+        run_name = ""
+        experiment_name = ""
+
+        delim_idx = name.find("/")
+        # just assume that name is the run_name (experiment name should default)
+        if delim_idx < 0:
+            run_name = name
+        elif delim_idx >= 0 and delim_idx < len(name) - 1:
+            # deal with:
+            # 1. /FOO (only run name)
+            # 2. FOO/BAR (both exp and run name)
+            #
+            # FOO/ (only exp name) will not enter this branch
+            # and end up getting an empty run_name (as declared above)
+            run_name = name[delim_idx + 1 :]
+
+        if delim_idx > 0:
+            experiment_name = name[:delim_idx]
+
+        if not run_name:
+            if m:  # use the last module name
+                run_name = m.rpartition(".")[2]
+            else:  # use script name w/ no extension
+                run_name = Path(none_throws(script)).stem
+        return StructuredNameArgument(
+            experiment_name or default_experiment_name, run_name
+        )
+
+
+@dataclass
+class StructuredJArgument:
+    nnodes: int
+    nproc_per_node: int
+
+    def __str__(self) -> str:
+        return f"{self.nnodes}x{self.nproc_per_node}"
+
+    @staticmethod
+    def parse_from(h: str, j: str) -> "StructuredJArgument":
+        """
+        Creates an :py:class:`ArgJ` instance given the ``h`` (host) and ``j`` (nnodes x nproc_per_node)
+        component arguments.
+
+        If the host has GPUs and ``j`` only specified nnodes (e.g. ``-j 2`` versus ``-j 2x8``), then
+        nproc_per_node is set equal to the number of GPUs on the host. If nproc_per_node was explicitly
+        specified, then it is honored even if it does not match the number of GPUs on the host.
+        However, a warning message is displayed reminding the user that there is a mismatch between
+        the GPU count on the host and the configured nproc_per_node.
+
+        Example (GPU):
+
+            .. doctest::
+
+                >>> from torchx.components.structured_arg import StructuredJArgument
+                >>> str(StructuredJArgument.parse_from(h="aws_p4d.24xlarge", j="2"))
+                '2x8'
+
+                >>> str(StructuredJArgument.parse_from(h="aws_p4d.24xlarge", j="2x4"))
+                '2x4'
+
+
+        For hosts with no GPU devices, one MUST specify nproc_per_node. Otherwise this function will
+        raise an Error.
+
+        Example (CPU or Trainium):
+
+            .. doctest::
+
+                >>> str(StructuredJArgument.parse_from(h="aws_trn1.32xl", j="2"))
+                Traceback (most recent call last):
+                    ...
+                ValueError: nproc_per_node cannot be inferred from GPU count. `trn1.32xl` is not a GPU instance. ...
+
+                >>> str(StructuredJArgument.parse_from(h="aws_trn1.32xl", j="2x16"))
+                '2x16'
+
+        """
+        nums = j.split("x")
+        num_gpus = specs.named_resources[h].gpu
+        if len(nums) == 1:  # -j 1
+            nnodes = int(nums[0])
+            # infer nproc_per_node from # of gpus in host
+
+            if num_gpus > 0:
+                nproc_per_node = num_gpus
+            else:
+                raise ValueError(
+                    f"nproc_per_node cannot be inferred from GPU count."
+                    f" `{h}` is not a GPU instance."
+                    f" You must specify `-j $NNODESx$NPROCS_PER_NODE` (e.g. `-j {nnodes}x8`)"
+                )
+
+        elif len(nums) == 2:  # -j 1x2
+            nnodes = int(nums[0])
+            nproc_per_node = int(nums[1])
+
+            if nproc_per_node != num_gpus:
+                warnings.warn(
+                    f"In `-j {j}` you specified nproc_per_node={nproc_per_node}"
+                    f" which does not equal the number of GPUs on a {h}: {num_gpus}."
+                    f" This may lead to under-utilization or an error. "
+                    f" If this was intentional, ignore this warning."
+                    f" Otherwise set `-j {nnodes}` to auto-set nproc_per_node"
+                    f" to the number of GPUs on the host."
+                )
+        else:
+            raise ValueError(
+                f"Invalid format for `-j $NNODESx$NPROCS_PER_NODE` (e.g. `-j 1x8`). Given: {j}"
+            )
+
+        return StructuredJArgument(nnodes=nnodes, nproc_per_node=nproc_per_node)

torchx/distributed/__init__.py

@@ -0,0 +1,270 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Convenience methods to use ``torch.distributed``.
+"""
+
+import logging
+import os
+import warnings
+from contextlib import contextmanager
+from typing import Any, Iterator
+
+import torch
+import torch.distributed as dist
+from typing_extensions import Literal
+
+log: logging.Logger = logging.getLogger(__name__)
+
+
+def local_rank() -> int:
+    """
+    Returns the local rank (aka rank within the node) of this process.
+    Typically the local rank is used to set the CUDA device on the node.
+
+    .. warning::
+        This function only works correctly if the invoker of the program sets ``LOCAL_RANK`` env var
+        or invokes the program with ``torchrun`` (aka ``torch.distributed.run``) or ``torchx``.
+        If ``LOCAL_RANK`` is not set or the process group is not initialized
+        then this function assumes that the process is not distributed and trivially returns 0.
+
+    """
+
+    if not dist.is_initialized():
+        return 0
+
+    if "LOCAL_RANK" not in os.environ:
+        warnings.warn(
+            "\n"
+            "==============================================================================================\n"
+            "`LOCAL_RANK` environment variable is not set. Will trivially return 0 for local_rank.\n"
+            " It is recommended to use torchrun/torchx to run your script or set the `LOCAL_RANK` manually.\n"
+            " For additional details see:\n"
+            "  1) https://pytorch.org/torchx/latest/components/distributed.html\n"
+            "  2) https://pytorch.org/docs/stable/elastic/run.html\n"
+            "=============================================================================================="
+        )
+        return 0
+    else:
+        return int(os.environ["LOCAL_RANK"])
+
+
+def local_cuda_device() -> torch.device:
+    """
+    Returns the CUDA device (as a ``torch.device``) based on the local rank.
+
+    See Also: :py:func:`get_local_rank`.
+    """
+    return torch.device(f"cuda:{local_rank()}")
+
+
+def rank() -> int:
+    """
+    A non-distributed-safe get_rank call. Unlike ``torch.distributed.get_rank()``
+    this method will not fail if being invoked from a non-distributed (e.g. process group not initialized)
+    context. Therefore, this method is safe to use in internal methods that may be used
+    in non-distributed contexts as well.
+
+    Returns:
+        If a process group has been initialized returns the value returned by ``torch.distributed.get_rank()``.
+        Otherwise, returns 0 (trivial rank)
+
+    """
+    return dist.get_rank() if dist.is_initialized() else 0
+
+
+def world_size() -> int:
+    """
+    A non-distributed-safe get_world_size call. Unlike ``torch.distributed.get_world_size()``,
+    this method will not fail if being invoked from a non-distributed (e.g. process group not initialized)
+    context. Threefore, this method is safe to use in internal mthods that may be used
+    in non-distributed contexts as well.
+
+    Returns:
+        If a process group has been initialized returns the value returns by ``torch.distributed.get_world_size()``.
+        Otherwise, returns 1 (trivial world_size)
+
+    """
+    return dist.get_world_size() if dist.is_initialized() else 1
+
+
+def is_rank0() -> bool:
+    """
+    Returns ``True`` if the caller is rank 0 (in a distributed setting).
+    If no process group has been initialized, then this method assumes
+    that the caller is a single-process (aka not-distributed) and trivially returns ``True``.
+    That is, for a non-distributed job there is only one process and hence that process
+    is trivially rank 0.
+
+    .. note::
+      To initialize the process group prefer to use :py:func:init_process_group over
+      ``torch.distributed.init_process_group()`` since the former can be called from
+      both distributed and non-distributed scripts.
+
+    """
+    return rank() == 0
+
+
+def is_local_rank0() -> bool:
+    """
+    Returns ``True`` if this process is local rank 0 and ``False`` otherwise.
+    Used to perform an action just once per node. Example
+
+    .. code-block:: python
+
+     if is_local_rank0():
+        # download a file just once per node
+        download_file("s3://...")
+
+
+    """
+    return local_rank() == 0
+
+
+Backend = Literal["nccl", "gloo", "auto"]
+
+
+def init_pg(backend: Backend = "auto", **kwargs: Any) -> torch.device:
+    """
+    A convenience wrapper around ``torch.distributed.init_proces_group()``
+    that makes initializing a trivial (single world_size) process group easy.
+
+    Useful when you want to make your code portable across launching with
+    simple python or with ``torchrun`` (aka ``torch.distributed.run``)
+
+    Usage:
+
+
+    .. doctest::
+
+     >>> from torchx.distributed import init_pg
+     >>> init_pg(backend="gloo") # or nccl # doctest: +SKIP
+     device(type='cpu')
+
+    The example above works to initialize a pytorch process group
+    for the trivial (``world_size = 1``) and distributed (``world_size > 1``)
+    cases without you having to write an explicit check with an if-else branch statement.
+
+    You can pass ``backend="auto"`` to have this function select ``"nccl"``
+    if there is a cuda device available, otherwise ``"gloo"`` (for CPU)
+
+
+    .. doctest::
+
+      >>> from torchx.distributed import init_pg
+      >>> device = init_pg(backend="auto") # doctest: +SKIP
+
+
+    In the code above, ``device`` will be ``cuda:{LOCAL_RANK}`` if the host has CUDA devices (GPUs)
+    and ``cpu`` if not.
+
+    Returns:
+        The cuda device that this rank should be using or cpu device if ``backend="gloo"``
+        or if ``backend="auto"`` and no GPUs are available on the host.
+
+    """
+
+    if backend == "auto":
+        backend = (
+            "nccl"
+            if torch.cuda.is_available()  # returns True if gpu-torch was installed even on CPU host
+            and (
+                torch.cuda.device_count() > 0
+            )  # so need to check for CUDA devices explicitly
+            and dist.is_nccl_available()
+            else "gloo"
+        )
+
+    # this means that the script was launched as a single python process
+    # initialize a trivial process group
+    if not dist.is_torchelastic_launched():
+        os.environ["MASTER_ADDR"] = "localhost"
+        os.environ["MASTER_PORT"] = "0"  # port selection - selects free random port
+        dist.init_process_group(backend=backend, rank=0, world_size=1, **kwargs)
+    else:
+        dist.init_process_group(backend=backend, **kwargs)
+
+    if backend == "nccl":
+        return local_cuda_device()
+    else:
+        return torch.device("cpu")
+
+
+@contextmanager
+def on_rank0_first() -> Iterator[None]:
+    """
+    Runs the piece of code that is wrapped in this context manager
+    first on rank0 then on the rest of the ranks.
+
+    Example:
+
+    .. code-block:: python
+
+        import time
+        from torchx.distributed import on_rank0_first, rank
+
+        with on_rank0_first():
+            print(f"Running on rank {rank()} at {int(time.monotonic())}")
+            time.sleep(10)
+
+
+    Would print:
+
+    .. code-block::
+
+        Running on rank 0 at 12534774
+        Running on rank 1 at 12534784 # at least +10 seconds on the other ranks
+        Running on rank 2 at 12534784
+        ...
+
+    To run ONLY on rank0 use an if-statement as such:
+
+    .. code-block:: python
+
+        if is_rank0():
+            print(f"Running on rank {dist.get_rank()}")
+
+    The code above would only print once on rank 0.
+
+    """
+    if dist.is_initialized() and not is_rank0():
+        dist.barrier()
+
+    try:
+        yield
+    finally:
+        if dist.is_initialized() and is_rank0():
+            dist.barrier()
+
+
+@contextmanager
+def on_local_rank0_first() -> Iterator[None]:
+    """
+    Runs the piece of code that is wrapped in this context manager
+    first on local rank 0 then on the rest of the ranks.
+
+    The behavior is exactly the same as :py:func:`torchx.distributed.on_rank0_first`
+    except that the barrier is on each local rank on each node (versus a global barrier on rank0).
+
+    This is useful in situations there a node-local action (that would otherwise cause races)
+    needs to be done first from a representative worker on each node. For instance,
+    downloading a checkpoint file to a tmp dir on each node once, then having all the
+    workers read off the downloaded file.
+
+    .. note::
+        For actions that need to be run first at a job level
+        use :py:func:`torchx.distributed.on_rank0_first`
+
+    """
+    if dist.is_initialized() and not is_local_rank0():
+        dist.barrier()
+
+    try:
+        yield
+    finally:
+        if dist.is_initialized() and is_local_rank0():
+            dist.barrier()

torchx/specs/__init__.py

@@ -14,6 +14,9 @@ import difflib
 from typing import Callable, Dict, Optional
 
 from torchx.specs.named_resources_aws import NAMED_RESOURCES as AWS_NAMED_RESOURCES
+from torchx.specs.named_resources_generic import (
+    NAMED_RESOURCES as GENERIC_NAMED_RESOURCES,
+)
 from torchx.util.entrypoints import load_group
 
 from .api import (  # noqa: F401 F403
@@ -55,12 +58,16 @@ GiB: int = 1024
 def _load_named_resources() -> Dict[str, Callable[[], Resource]]:
     resource_methods = load_group("torchx.named_resources", default={})
     materialized_resources: Dict[str, Callable[[], Resource]] = {}
-    default = AWS_NAMED_RESOURCES
-    for name, resource in default.items():
+
+    for name, resource in {
+        **GENERIC_NAMED_RESOURCES,
+        **AWS_NAMED_RESOURCES,
+        **resource_methods,
+    }.items():
         materialized_resources[name] = resource
-    for resource_name, resource_method in resource_methods.items():
-        materialized_resources[resource_name] = resource_method
+
     materialized_resources["NULL"] = lambda: NULL_RESOURCE
+    materialized_resources["MISSING"] = lambda: NULL_RESOURCE
     return materialized_resources
 
 

torchx/specs/named_resources_generic.py

@@ -0,0 +1,58 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Defines generic named resources that are not specific to any cloud provider's
+instance types. These generic named resources are meant to be used as
+default values for components and examples and are NOT meant to be used
+long term as the specific capabilities (e.g. number of cpu, gpu, memMB)
+are subject to change.
+
+.. note:: T
+    he named resources in this file DO NOT map device capabilities such as
+    special network interfaces (e.g. EFA devices on AWS).
+.. warning::
+    Do not use for launching applications that require specific capabilities
+    (e.g. needs exactly 4 x A100 GPUs with 40GB of memory connected with NVLink).
+
+Different cloud provides offer different types of
+instance types hence practically speaking one should register their own
+named resources that accurately capture the instances they have at their disposal
+rather than using these defaults long term.
+
+.. note::
+    The cpu/gpu/memory ratios in these default resources are based on current
+    HW trends and do not map exactly to a particular instance type!
+
+.. warning::
+    The specific capabilities of these default resources are subject to change
+    at any time based on current hardware spec trends.
+    Therefore, the user should NEVER assume that the specific number of cpu, gpu, and memMB
+    will always remain the same. For instance, never assume that ``gpu.small`` will always
+    have 8 cpus.
+
+"""
+from typing import Callable, Mapping
+
+from torchx.specs.api import Resource
+
+GiB: int = 1024
+
+NAMED_RESOURCES: Mapping[str, Callable[[], Resource]] = {
+    # typically system CPU memory is >= GPU memory (most modern GPUs have 32GB device mem)
+    # most cloud provides offer 1, 2, 4, 8 GPUs per host
+    "gpu.small": lambda: Resource(cpu=8, gpu=1, memMB=32 * GiB),
+    "gpu.medium": lambda: Resource(cpu=16, gpu=2, memMB=64 * GiB),
+    "gpu.large": lambda: Resource(cpu=32, gpu=4, memMB=128 * GiB),
+    "gpu.xlarge": lambda: Resource(cpu=64, gpu=8, memMB=256 * GiB),
+    # for cpu defaults - based on AWS's T2 (general purpose) instance type
+    "cpu.nano": lambda: Resource(cpu=1, gpu=0, memMB=512),
+    "cpu.micro": lambda: Resource(cpu=1, gpu=0, memMB=1 * GiB),
+    "cpu.small": lambda: Resource(cpu=1, gpu=0, memMB=2 * GiB),
+    "cpu.medium": lambda: Resource(cpu=2, gpu=0, memMB=4 * GiB),
+    "cpu.large": lambda: Resource(cpu=2, gpu=0, memMB=8 * GiB),
+    "cpu.xlarge": lambda: Resource(cpu=8, gpu=0, memMB=32 * GiB),
+}

torchx/tracker/mlflow.py

@@ -0,0 +1,342 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import dataclasses
+import os
+import socket
+from getpass import getuser
+from logging import getLogger, Logger
+from pathlib import Path
+from tempfile import gettempdir
+from typing import Any, Dict, Iterable, Mapping, Optional, Sequence
+
+import mlflow
+from mlflow import MlflowClient
+from mlflow.entities import Experiment, Run
+
+from torchx.distributed import on_rank0_first
+from torchx.runner.config import get_configs
+from torchx.tracker.api import Lineage, TrackerArtifact, TrackerBase, TrackerSource
+
+log: Logger = getLogger(__name__)
+TAG_ARTIFACT_MD_PREFIX = "torchx.artifact.metadata"
+
+
+class MLflowTracker(TrackerBase):
+    """
+    An implementation of a ``Tracker`` that uses mlflow as the backend.
+    Don't forget to call the ``close()`` method for orderly shutdown.
+    This ensures that the run state in mlflow is properly marked as ``FINISHED``,
+    otherwise the run will remain in ``UNFINISHED`` status.
+
+    .. important::
+        TorchX's run_id is used as mlflow's run_name! The run_id in TorchX
+        is the job name. The job name in TorchX is made unique by adding
+        a short random hash to the user-provided job name prefix. This is
+        done because certain job schedulers supported by TorchX requires
+        that the job name on the submitted job definition is globally unique
+        (rather than the scheduler returning a unique job id as the return result
+        of the job submission API).
+
+    .. warning::
+        APIs on this class may only be called with the same ``run_name`.
+        Typically the user does not have to worry about manually setting
+        the run_name as it is picked up by default from the environment variable
+        ``TORCHX_APP_NAME``.
+
+    """
+
+    def __init__(
+        self,
+        experiment_name: Optional[str] = None,
+        tracking_uri: str = f"file://{Path(gettempdir()) / 'torchx' / 'mlruns'}",
+        artifact_location: Optional[str] = None,
+    ) -> None:
+        if experiment_name is None:
+            experiment_name = self.default_experiment_name()
+
+        self.tracking_uri = tracking_uri
+        mlflow.set_tracking_uri(tracking_uri)
+        log.info(
+            f"MLflow tracking_uri={tracking_uri}, artifact_location={artifact_location}"
+        )
+        with on_rank0_first():
+            existing_experiment = mlflow.get_experiment_by_name(experiment_name)
+            if existing_experiment:
+                self.experiment: Experiment = existing_experiment
+                log.info(
+                    f"Found existing experiment `{experiment_name}` (id={self.experiment_id})"
+                )
+            else:
+                experiment_id = mlflow.create_experiment(
+                    name=experiment_name,
+                    artifact_location=artifact_location,
+                )
+                self.experiment = mlflow.get_experiment(experiment_id)
+                log.info(
+                    f"Created new experiment `{experiment_name}` (id={experiment_id})"
+                )
+
+    @staticmethod
+    def default_experiment_name() -> str:
+        return f"default-experiment/{getuser()}/{socket.getfqdn()}"
+
+    @property
+    def experiment_id(self) -> str:
+        return self.experiment.experiment_id
+
+    @property
+    def experiment_name(self) -> str:
+        return self.experiment.name
+
+    @property
+    def artifact_location(self) -> str:
+        return self.experiment.artifact_location
+
+    def get_run(self, run_name: str) -> Run:
+        """
+        Gets mlflow's ``Run`` object for the given ``run_name`` in the current experiment.
+        If no such run exists, this method creates a new run under
+        this experiment and starts the run so that subsequent calls to
+        mlflow logs metadata, metrics, artifacts to the newly created run.
+
+        .. warning::
+            This method should only be called with the same run_name!
+            This is because of the way mlflow
+            APIs work is by setting an "active run" for which subsequent
+            mlflow logging APIs are made against the current active run
+            in the stack. If you call ``mlflow.start_run()`` directly
+            or pass different run names, then you may be logging into two different
+            mlflow runs from the same job!
+
+        Args:
+            run_name: equal to torchx's run_id
+
+        Returns: mlflow's ``Run`` object for the ``run_name``
+
+        """
+
+        active_run = mlflow.active_run()
+
+        if active_run is None:
+            search_result = mlflow.search_runs(
+                experiment_ids=[self.experiment_id],
+                output_format="list",
+                filter_string=f"tags.`mlflow.runName` = '{run_name}'",
+            )
+            if not search_result:
+                return mlflow.start_run(
+                    experiment_id=self.experiment_id, run_name=run_name
+                )
+            elif len(search_result) == 1:
+                return search_result[0]
+            else:  # len(search_result) > 1
+                raise RuntimeError(
+                    f"More than 1 run found for run_name `{run_name}` in experiment `{self.experiment_name}`."
+                    f" Did you manually create runs with the same name under this experiment?"
+                    f" Remove duplicate run names and try again"
+                )
+        else:
+            # need to query mlflow again so that the run reflects any newly written logs
+            return mlflow.get_run(active_run.info.run_id)
+
+    def get_run_id(self, run_name: str) -> str:
+        """
+        Gets the mlflow run's run_id for the given ``run_name`` and additionally sets
+        this run as the active run. Hence this method has a side-effect where all subsequent
+        calls to mlflow log APIs are against the run for the given ``run_name``.
+        """
+        return self.get_run(run_name).info.run_id
+
+    def close(self) -> None:
+        mlflow.end_run()
+
+    def add_artifact(
+        self,
+        run_id: str,
+        name: str,
+        path: str,
+        metadata: Optional[Mapping[str, object]] = None,
+    ) -> None:
+        self.get_run(run_id)
+        # stores the artifact in {artifact_location}/{name} (e.g. s3://bucket/prefix/{name})
+        mlflow.log_artifact(local_path=path, artifact_path=name)
+
+        # add artifact metadata with torchx.artifact_metadata.{name}.* tag prefix
+        if metadata:
+            mlflow.set_tags(
+                tags={
+                    f"{TAG_ARTIFACT_MD_PREFIX}.{name}.{k}": v
+                    for k, v in metadata.items()
+                }
+            )
+
+    def artifacts(self, run_id: str) -> Mapping[str, TrackerArtifact]:
+        artifacts: Dict[str, TrackerArtifact] = {}
+        mlflow_client: MlflowClient = MlflowClient(self.tracking_uri)
+
+        def get_artifacts(path: Optional[str] = None) -> None:
+            for artifact_info in mlflow_client.list_artifacts(
+                self.get_run(run_id).info.run_id, path=path
+            ):
+                if artifact_info.is_dir:
+                    get_artifacts(path=artifact_info.path)
+                else:
+                    # we stored the artifact using the name as the path
+                    # so path should never be `None` when we get to this point
+                    # (e.g. the root of `artifact_location` will only have directories
+                    # where the directory names are the artifact names
+                    name = path or "<SHOULD_NOT_HAPPEN>"
+
+                    # artifact metadata is stored as run tags with `torchx.artifact.metadata.*` prefix
+                    tag_prefix = f"{TAG_ARTIFACT_MD_PREFIX}.{name}."
+                    metadata = {
+                        # k.removeprefix() only avail in python 3.9+
+                        k[len(tag_prefix) :]: v
+                        for k, v in self.metadata(run_id).items()
+                        if k.startswith(tag_prefix)
+                    }
+
+                    # add some additional metadata about the artifact
+                    metadata["mlflow.file_size"] = artifact_info.file_size
+
+                    artifacts[name] = TrackerArtifact(
+                        name=name,
+                        path=f"{self.artifact_location}/{artifact_info.path}",
+                        metadata=metadata,
+                    )
+
+        get_artifacts()
+        return artifacts
+
+    def add_metadata(self, run_id: str, **kwargs: object) -> None:
+        self.get_run(run_id)
+        mlflow.set_tags(tags={k: v for k, v in kwargs.items()})
+
+    def metadata(self, run_id: str) -> Mapping[str, object]:
+        return self.get_run(run_id).data.tags
+
+    def run_ids(self, **kwargs: str) -> Iterable[str]:
+        runs = mlflow.search_runs(
+            experiment_ids=[self.experiment_id], output_format="list"
+        )
+        return [r.info.run_name for r in runs]
+
+    def log_params_flat(
+        self, run_name: str, cfg: Any, key: str = ""  # pyre-ignore[2]
+    ) -> None:
+        """
+        Designed to be primarily used with hydra-style config objects (e.g. dataclasses),
+        logs the given ``cfg``, which is one of: ``@dataclass``, ``Sequence`` (e.g. ``list``, ``tuple``, ``set``),
+        or ``Mapping`` (e.g. ``dict``), where the fields of ``cfg`` are flattened recursively and logged as
+        the the run's ``Parameter`` in mlflow.
+
+        For example if ``cfg`` is:
+
+        .. code-block:: python
+
+            @dataclass
+            class Config2:
+                foo: str = "bar"
+
+            @dataclass
+            class Config:
+                i: int = 1
+                f: float = 2.1
+                s: str = "string"
+                l: List[str] = field(default_factory=lambda :["a", "b", "c"])
+                cfg_list = List[Config2] = field(default_factory=lambda : [Config2(foo="hello"), Config2(foo="world")])
+                cfg2: Config2 = Config2()
+
+
+        Then this function logs the following parameters
+
+        .. code-block::
+
+            i: "1"
+            f: "2.1"
+            s: "string"
+            l: ["a", "b", "c"]
+            cfg_list._0.foo = "hello"
+            cfg_list._1.foo = "hello"
+            cfg2.foo = "bar"
+
+        As shown above, primitive sequence containers are logged directly (e.g. ``l: ["a", "b", "c"]``)
+        whereas nested sequence containers will be logged per element where the key is suffixed with
+        ``_{INDEX}`` (e.g. ``cfg_list._0.foo = "hello"``).
+
+        """
+        if dataclasses.is_dataclass(cfg):
+            cfg = dataclasses.asdict(cfg)
+        self.get_run(run_name)
+
+        def is_primitive(v: Any) -> bool:  # pyre-ignore[2]
+            return isinstance(v, (str, int, float, bool))
+
+        key_prefix = f"{key}." if key else ""
+
+        if not cfg:
+            # empty container; log as is
+            mlflow.log_param(key, cfg)
+        else:
+            # non-empty container; check types
+            if isinstance(cfg, (Sequence, set)):
+                # assume list/set elements are homogeneous
+                # need only check first element for type
+                elem = next(iter(cfg))
+                if is_primitive(elem):
+                    mlflow.log_param(key, cfg)
+                else:
+                    for i, e in enumerate(cfg):
+                        self.log_params_flat(run_name, e, f"{key_prefix}_{i}")
+            elif isinstance(cfg, Mapping):
+                for k, v in cfg.items():
+                    if is_primitive(v):
+                        mlflow.log_param(f"{key_prefix}{k}", v)
+                    else:
+                        self.log_params_flat(run_name, v, f"{key_prefix}{k}")
+
+    def add_source(
+        self, run_id: str, source_id: str, artifact_name: Optional[str] = None
+    ) -> None:
+        raise NotImplementedError(
+            f"Job's tracker sources is currently unsupported for {self.__class__.__qualname__}"
+        )
+
+    def sources(
+        self, run_id: str, artifact_name: Optional[str] = None
+    ) -> Iterable[TrackerSource]:
+        raise NotImplementedError(
+            f"Job's tracker sources is currently unsupported for {self.__class__.__qualname__}"
+        )
+
+    def lineage(self, run_id: str) -> Lineage:
+        raise NotImplementedError(
+            f"Job's lineage is currently unsupported for {self.__class__.__qualname__}"
+        )
+
+
+def create_tracker(config: str) -> MLflowTracker:
+    ctor_args = get_configs(
+        prefix="tracker",
+        name="mlflow",
+        dirs=[config],
+    )
+
+    # remove "config" key since that one is reserved for torchx.tracker usage
+    ctor_args.pop("config", None)
+
+    # pass configs read from .torchxconfig [tracker:mlflow] section as kwargs
+    # get the experiment name from an env var (set in torchx.components.dist:spmd)
+    # if no such env var exists, then default the experiment_name to the one
+    # specified in .torchxconfig
+    return MLflowTracker(
+        experiment_name=os.getenv(
+            "TORCHX_TRACKING_EXPERIMENT_NAME",
+            default=ctor_args.pop("experiment_name", None),
+        ),
+        **ctor_args,
+    )

torchx/util/strings.py

@@ -10,7 +10,7 @@ import re
 def normalize_str(data: str) -> str:
     """
     Invokes ``lower`` on thes string and removes all
-    characters that do not satisfy ``[a-z0-9]`` pattern.
+    characters that do not satisfy ``[a-z0-9\\-]`` pattern.
     This method is mostly used to make sure kubernetes and gcp_batch scheduler gets
     the job name that does not violate its restrictions.
     """

{torchx_nightly-2023.3.21.dist-info → torchx_nightly-2023.3.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: torchx-nightly
-Version: 2023.3.21
+Version: 2023.3.22
 Summary: TorchX SDK and Components
 Home-page: https://github.com/pytorch/torchx
 Author: TorchX Devs
@@ -42,6 +42,7 @@ Requires-Dist: google-cloud-runtimeconfig (>=0.33.2) ; extra == 'dev'
 Requires-Dist: hydra-core ; extra == 'dev'
 Requires-Dist: ipython ; extra == 'dev'
 Requires-Dist: kfp (==1.8.9) ; extra == 'dev'
+Requires-Dist: mlflow-skinny ; extra == 'dev'
 Requires-Dist: moto (==4.1.3) ; extra == 'dev'
 Requires-Dist: pyre-extensions ; extra == 'dev'
 Requires-Dist: pyre-check ; extra == 'dev'

{torchx_nightly-2023.3.21.dist-info → torchx_nightly-2023.3.22.dist-info}/RECORD

@@ -24,15 +24,17 @@ torchx/cli/colors.py,sha256=bVN_jEDwLgvypnDMeCHKn0q0ZDDhQjBJnyVfZHAE6nc,553
 torchx/cli/main.py,sha256=ysAQh0vPn0hC3JeVzRP0i7-E6dTO2D4rLN2B5Ok3abw,3442
 torchx/components/__init__.py,sha256=6-TQ4SY-Tn56os_1lOs_HMabOoE7gkkud_8e1BgvfJw,12106
 torchx/components/component_test_base.py,sha256=eKOwBp5cRgiA4FgZd_FCvyJ-ppv2v3JN9AGXnaSK_Cw,4135
-torchx/components/dist.py,sha256=l_IEtblhTTu-_8lDpfWNo5Pc_9teE9bVWDhRy_3hJ7s,9931
+torchx/components/dist.py,sha256=FA1Wxxdo4qZ1_zd2Y20JpHog2ZjEGCkjN0k7dz5ZLvE,14075
 torchx/components/interpret.py,sha256=g8gkKdDJvsBfX1ZrpVT7n2bMEtmwRV_1AqDyAnnQ_aA,697
 torchx/components/metrics.py,sha256=1gbp8BfzZWGa7PD1db5vRADlONzmae4qSBUUdCWayr0,2814
 torchx/components/serve.py,sha256=9RlpwlU2KOC7sMOZBeYwUpJIKDCXrU8xNo1SH-AT3fc,2141
+torchx/components/structured_arg.py,sha256=uavcUeFDRnMP7cWAqcxR3ujJYi6JEsClz0_Rd4Dgxj4,9542
 torchx/components/train.py,sha256=vtrQXRcD7bIcbb3lSeyD9BBlIe1mv1WNW6rnLK9R0Mw,1259
 torchx/components/utils.py,sha256=m7mFe6du2AMHpViwcW9dF8rr_twQB6KHQuEzJyHwBXw,9025
 torchx/components/integration_tests/__init__.py,sha256=Md3cCHD7Ano9kV15PqGbicgUO-RMdh4aVy1yKiDt_xE,208
 torchx/components/integration_tests/component_provider.py,sha256=fcdVWJt6dewv16qHOLl4PKQUK1PSfkPWy7d4dKMqZmo,3925
 torchx/components/integration_tests/integ_tests.py,sha256=OVgRvGrLWhDUNlqbbYj90ukGmkAwka2KubCWUR8pC7Y,5150
+torchx/distributed/__init__.py,sha256=OAO1CIwVBOaclzbp2NjH_SMBq2WlK7aE9NVlNmDtVlQ,8786
 torchx/examples/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 torchx/examples/apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 torchx/examples/apps/datapreproc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -78,12 +80,13 @@ torchx/schedulers/streams.py,sha256=ObaKwEEcnsjrPyc6VZOp8cgZ_f2RFextAxeISxZUWeQ,
 torchx/schedulers/ray/__init__.py,sha256=fE0IHi1JJpxsNVBNzWNee2thrNXFFRhY94c80RxNSIE,231
 torchx/schedulers/ray/ray_common.py,sha256=pyNYFvTKVwdjDAeCBNbPwAWwVNmlLOJWExfn90XY8u8,610
 torchx/schedulers/ray/ray_driver.py,sha256=0DL8Ad_hire-WgH8ZEYx1Q-mI2SUfZDk-6_6PICk8OQ,12282
-torchx/specs/__init__.py,sha256=lWdg9f9OVtJ5QmgMO_o6ek9N3gTb8XimP0HIlJVrZcY,5369
+torchx/specs/__init__.py,sha256=fSA89Y0ZpdZLJmhIfEKNbjNNi6fbDR9k1bpIM7Xm7xo,5462
 torchx/specs/api.py,sha256=vFtvhYW18HvWzET8ob8ONNRC1MCuYwLMXohWDy0LbZI,33798
 torchx/specs/builders.py,sha256=dsKa80PD-cuIjXhFF2TsEY0eL8S01wux8aZaQrURIEE,8512
 torchx/specs/file_linter.py,sha256=LREWELpHJyE7YN3rc5ixf2ZydWFU9dlcSy5gGqdB5rA,11714
 torchx/specs/finder.py,sha256=RJI0PkG69esuVzhCp4w6Lotu2tSzIRh6PhWemCSQR7I,16234
 torchx/specs/named_resources_aws.py,sha256=6ID0jOGi5HVXn9BLylzCMD-SUQtxGeBHS5zU8PCten4,6361
+torchx/specs/named_resources_generic.py,sha256=_xz0cRjy3fz-CVtX9G_MY7f3NX6n3AkP3xzAkuDevwk,2631
 torchx/specs/test/components/__init__.py,sha256=Md3cCHD7Ano9kV15PqGbicgUO-RMdh4aVy1yKiDt_xE,208
 torchx/specs/test/components/a/__init__.py,sha256=T7exlQ47Fak5ajCEGPg6_yOfChJCWpIMhWBmSVUnlrQ,546
 torchx/specs/test/components/a/b/__init__.py,sha256=Md3cCHD7Ano9kV15PqGbicgUO-RMdh4aVy1yKiDt_xE,208
@@ -92,6 +95,7 @@ torchx/specs/test/components/c/__init__.py,sha256=fE0IHi1JJpxsNVBNzWNee2thrNXFFR
 torchx/specs/test/components/c/d.py,sha256=RH07jjo6uvFbzIaNFnAwmD_h24cEsT8kyZDTN-ezFio,531
 torchx/tracker/__init__.py,sha256=kaynfAwMYtdkkvXNrb4Rmin2mSd5vi9mg25iz1e0TJ8,4200
 torchx/tracker/api.py,sha256=qIgoHNlc_RsQbQdcHF8GDrKby_vLlZg84Tn6VfONrY4,11284
+torchx/tracker/mlflow.py,sha256=P_mj7Yi-bZc7QOZ-6PJW4FHeWWGX0rjauNK0zXC25ig,13221
 torchx/tracker/backend/__init__.py,sha256=fE0IHi1JJpxsNVBNzWNee2thrNXFFRhY94c80RxNSIE,231
 torchx/tracker/backend/fsspec.py,sha256=JpSioMgn54mrxqqpY0kw5Gudqx9hhxkgDLaOFSEP2Ko,10425
 torchx/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -99,15 +103,15 @@ torchx/util/datetime.py,sha256=e-sO5Wjx1Utpln14C3qfJHl4v3KM-SMnn11hSyvkqFY,390
 torchx/util/entrypoints.py,sha256=C4A7cF1tPLlfyYWyZ7uZEtsKeuoOoLbMv0sOSxLhXs4,2710
 torchx/util/io.py,sha256=sxb6KI42Lq6n5z6_-YKW_mAhgPdC6CxzexlMyGheWSc,1792
 torchx/util/shlex.py,sha256=KzyWektMeU3oXS3Z5mFkNSPLItBTszVcvQ3EYfOMUYA,448
-torchx/util/strings.py,sha256=CfR2FtT0bNFQW-jeILL5gr85u3QShpBDR5_VIJmyYI0,660
+torchx/util/strings.py,sha256=7CZe5WKHa7IQ6DuJCYeJ5FapUC4Fd1OGeq1yZAmjluw,663
 torchx/util/types.py,sha256=6ASuDKGO91UU3DCSuWhPX_C03341tApLCQEByUz8xpY,7016
 torchx/workspace/__init__.py,sha256=KbGEzJqqXaIxALm_EQO64aw-fE7MeDMFXcpU1mY650I,783
 torchx/workspace/api.py,sha256=Ej6DR__mNWaVyZgoVNAAOloDy1kTD5X1jz7pRtoVf80,5464
 torchx/workspace/dir_workspace.py,sha256=Fz-hKIx0KN8iJf2BsthNj0NvTkWlxP6WFsElPs_BaT0,2253
 torchx/workspace/docker_workspace.py,sha256=Yd8ut26bNfjyJQnmH8ANOrflfr-4VKcnOrIjbi_XIUY,9208
-torchx_nightly-2023.3.21.dist-info/LICENSE,sha256=e0Eotbf_rHOYPuEUlppIbvwy4SN98CZnl_hqwvbDA4Q,1530
-torchx_nightly-2023.3.21.dist-info/METADATA,sha256=hOzOlBCLCSFXHNM2iUTNArDVurcsIOvSIURfK3OrBdM,5376
-torchx_nightly-2023.3.21.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-torchx_nightly-2023.3.21.dist-info/entry_points.txt,sha256=3JYZFlX9aWzR-Gs_qsx1zq7mlqbFz6Mi9rQUULW8caI,170
-torchx_nightly-2023.3.21.dist-info/top_level.txt,sha256=pxew3bc2gsiViS0zADs0jb6kC5v8o_Yy_85fhHj_J1A,7
-torchx_nightly-2023.3.21.dist-info/RECORD,,
+torchx_nightly-2023.3.22.dist-info/LICENSE,sha256=e0Eotbf_rHOYPuEUlppIbvwy4SN98CZnl_hqwvbDA4Q,1530
+torchx_nightly-2023.3.22.dist-info/METADATA,sha256=UbHY2VUh7IvI4ty1vY1HD-0V8LCtrIakJVG77fVzEaU,5422
+torchx_nightly-2023.3.22.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+torchx_nightly-2023.3.22.dist-info/entry_points.txt,sha256=3JYZFlX9aWzR-Gs_qsx1zq7mlqbFz6Mi9rQUULW8caI,170
+torchx_nightly-2023.3.22.dist-info/top_level.txt,sha256=pxew3bc2gsiViS0zADs0jb6kC5v8o_Yy_85fhHj_J1A,7
+torchx_nightly-2023.3.22.dist-info/RECORD,,