torchx-nightly 2024.10.15__py3-none-any.whl → 2024.10.17__py3-none-any.whl

torchx/runner/api.py

@@ -14,7 +14,7 @@ import time
 import warnings
 from datetime import datetime
 from types import TracebackType
-from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, Type
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, Type, TypeVar
 
 from torchx.runner.events import log_event
 from torchx.schedulers import get_scheduler_factories, SchedulerFactory
@@ -41,7 +41,7 @@ from torchx.tracker.api import (
 )
 
 from torchx.util.types import none_throws
-from torchx.workspace.api import WorkspaceMixin
+from torchx.workspace.api import PkgInfo, WorkspaceBuilder, WorkspaceMixin
 
 from .config import get_config, get_configs
 
@@ -49,6 +49,8 @@ logger: logging.Logger = logging.getLogger(__name__)
 
 
 NONE: str = "<NONE>"
+S = TypeVar("S")
+T = TypeVar("T")
 
 
 def get_configured_trackers() -> Dict[str, Optional[str]]:
@@ -147,6 +149,18 @@ class Runner:
         for scheduler in self._scheduler_instances.values():
             scheduler.close()
 
+    def build_standalone_workspace(
+        self,
+        workspace_builder: WorkspaceBuilder[S, T],
+        sync: bool = True,
+    ) -> PkgInfo[S]:
+        """
+        Build a standalone workspace for the given role.
+        This method is used to build a workspace for a role independent of the scheduler and
+        also enables asynchronous workspace building using the Role overrides.
+        """
+        return workspace_builder.build_workspace(sync)
+
     def run_component(
         self,
         component: str,

torchx/runner/events/__init__.py

@@ -20,7 +20,9 @@ Example of usage:
 
 """
 
+import json
 import logging
+import sys
 import time
 import traceback
 from types import TracebackType
@@ -123,6 +125,20 @@ class log_event:
         ) // 1000
         if traceback_type:
             self._torchx_event.raw_exception = traceback.format_exc()
+            typ, value, tb = sys.exc_info()
+            if tb:
+                last_frame = traceback.extract_tb(tb)[-1]
+                self._torchx_event.exception_source_location = json.dumps(
+                    {
+                        "filename": last_frame.filename,
+                        "lineno": last_frame.lineno,
+                        "name": last_frame.name,
+                    }
+                )
+        if exec_type:
+            self._torchx_event.exception_type = exec_type.__name__
+        if exec_value:
+            self._torchx_event.exception_message = str(exec_value)
         record(self._torchx_event)
 
     def _generate_torchx_event(

torchx/runner/events/api.py

@@ -52,6 +52,9 @@ class TorchxEvent:
     wall_time_usec: Optional[int] = None
     start_epoch_time_usec: Optional[int] = None
     workspace: Optional[str] = None
+    exception_type: Optional[str] = None
+    exception_message: Optional[str] = None
+    exception_source_location: Optional[str] = None
 
     def __str__(self) -> str:
         return self.serialize()

torchx/specs/api.py

@@ -11,6 +11,7 @@ import asyncio
 import copy
 import inspect
 import json
+import logging as logger
 import re
 import typing
 from dataclasses import asdict, dataclass, field
@@ -189,7 +190,16 @@ class macros:
             apply applies the values to a copy the specified role and returns it.
             """
 
+            # Overrides might contain future values which can't be serialized so taken out for the copy
+            overrides = role.overrides
+            if len(overrides) > 0:
+                logger.warning(
+                    "Role overrides are not supported for macros. Overrides will not be copied"
+                )
+                role.overrides = {}
             role = copy.deepcopy(role)
+            role.overrides = overrides
+
             role.args = [self.substitute(arg) for arg in role.args]
             role.env = {key: self.substitute(arg) for key, arg in role.env.items()}
             role.metadata = self._apply_nested(role.metadata)

torchx/workspace/api.py

@@ -9,7 +9,8 @@
 import abc
 import fnmatch
 import posixpath
-from typing import Generic, Iterable, Mapping, Tuple, TYPE_CHECKING, TypeVar
+from dataclasses import dataclass
+from typing import Any, Dict, Generic, Iterable, Mapping, Tuple, TYPE_CHECKING, TypeVar
 
 from torchx.specs import AppDef, CfgVal, Role, runopts
 
@@ -20,6 +21,36 @@ TORCHX_IGNORE = ".torchxignore"
 
 T = TypeVar("T")
 
+PackageType = TypeVar("PackageType")
+WorkspaceConfigType = TypeVar("WorkspaceConfigType")
+
+
+@dataclass
+class PkgInfo(Generic[PackageType]):
+    """
+    Convenience class used to specify information regarding the built workspace
+    """
+
+    img: str
+    lazy_overrides: Dict[str, Any]
+    metadata: PackageType
+
+
+@dataclass
+class WorkspaceBuilder(Generic[PackageType, WorkspaceConfigType]):
+    cfg: WorkspaceConfigType
+
+    @abc.abstractmethod
+    def build_workspace(self, sync: bool = True) -> PkgInfo[PackageType]:
+        """
+        Builds the specified ``workspace`` with respect to ``img``.
+        In the simplest case, this method builds a new image.
+        Certain (more efficient) implementations build
+        incremental diff patches that overlay on top of the role's image.
+
+        """
+        pass
+
 
 class WorkspaceMixin(abc.ABC, Generic[T]):
     """

{torchx_nightly-2024.10.15.dist-info → torchx_nightly-2024.10.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: torchx-nightly
-Version: 2024.10.15
+Version: 2024.10.17
 Summary: TorchX SDK and Components
 Home-page: https://github.com/pytorch/torchx
 Author: TorchX Devs

{torchx_nightly-2024.10.15.dist-info → torchx_nightly-2024.10.17.dist-info}/RECORD

@@ -56,10 +56,10 @@ torchx/pipelines/kfp/__init__.py,sha256=8iJ8lql_fxwuk9VCYSxXnX6tPL228fB5mDZpOs-k
 torchx/pipelines/kfp/adapter.py,sha256=V96Rg2ypas0ZpNNn4ojplJMmC5vcBfcuxAITaY8PV_M,8969
 torchx/pipelines/kfp/version.py,sha256=mYBxd6bm4MeR34D--xo-JLQ9wHeAl_ZQLwbItCf9tr0,539
 torchx/runner/__init__.py,sha256=x8Sz7s_tLxPgJgvWIhK4ju9BNZU61uBFywGwDY6CqJs,315
-torchx/runner/api.py,sha256=17ZFEYeuIK0kUrTUysdgsFaEVc6_-ggvBuSoabQnrcA,28625
+torchx/runner/api.py,sha256=1FlElbo-j5M3n1eivb3vE1RaMCXEXexGWrAlS-Mbi3w,29153
 torchx/runner/config.py,sha256=fTdCcf-MKlBg6MzXopF4W0hYyDDoPAuvZs2v2bKzwG0,17849
-torchx/runner/events/__init__.py,sha256=TtzBLZ9oaHKfr689R4NnjCii3G8kxiRafe7Q0jRE5_k,4649
-torchx/runner/events/api.py,sha256=_6mjS5B5FcSMEea1U0M-fD4jErmbpZ1gQOyUHV-4dqY,2527
+torchx/runner/events/__init__.py,sha256=1_y0bojXl3FL0zlAj7BI4Dg5cXKXUmaa2jZbVH0EDUA,5268
+torchx/runner/events/api.py,sha256=pPLfowWTXtN_XcrEDNI45pE6Ijvdc_Gdxq76RduqgGE,2664
 torchx/runner/events/handlers.py,sha256=ThHCIJW21BfBgB7b6ftyjASJmD1KdizpjuTtsyqnvJs,522
 torchx/runtime/__init__.py,sha256=Wxje2BryzeQneFu5r6P9JJiEKG-_C9W1CcZ_JNrKT6g,593
 torchx/runtime/tracking/__init__.py,sha256=dYnAPnrXYREfPXkpHhdOFkcYIODWEbA13PdD-wLQYBo,3055
@@ -83,7 +83,7 @@ torchx/schedulers/ray/__init__.py,sha256=fE0IHi1JJpxsNVBNzWNee2thrNXFFRhY94c80Rx
 torchx/schedulers/ray/ray_common.py,sha256=pyNYFvTKVwdjDAeCBNbPwAWwVNmlLOJWExfn90XY8u8,610
 torchx/schedulers/ray/ray_driver.py,sha256=Wl-1jldL8veVKzmYDEeR2va3JSlAjZpFE1h8HWE9YVE,12286
 torchx/specs/__init__.py,sha256=T8xUCz7iVE6OsUL5P4Pzy2B8ZY_YinCVDwUer5Q-XPc,6179
-torchx/specs/api.py,sha256=ipALDCFY83vqZ47HxT73ZAQFhHvUcNnzVcJov2B8nzM,37115
+torchx/specs/api.py,sha256=EI1tXPfDrz8dAyjJsuNK9mL4femS44lUuJFJUCW_oSc,37540
 torchx/specs/builders.py,sha256=QDcQrnCO4bdSaiP0216XbCgTsnLutO_1_FW5jDiEIWI,9939
 torchx/specs/file_linter.py,sha256=IeiomB1BgHUlT-ZsvGxar3llY63NOupfLBrOrD_---A,11860
 torchx/specs/finder.py,sha256=MnwxG_UC4a-3X2wQ37ANEQR6D1TvriCLyuVYBh_-wuI,16249
@@ -112,12 +112,12 @@ torchx/util/shlex.py,sha256=eXEKu8KC3zIcd8tEy9_s8Ds5oma8BORr-0VGWNpG2dk,463
 torchx/util/strings.py,sha256=GkLWCmYS89Uv6bWc5hH0XwvHy7oQmprv2U7axC4A2e8,678
 torchx/util/types.py,sha256=een55pV-N8aBc3qUBjHRc1llJcX10JVa19pB8dBE8No,7564
 torchx/workspace/__init__.py,sha256=FqN8AN4VhR1C_SBY10MggQvNZmyanbbuPuE-JCjkyUY,798
-torchx/workspace/api.py,sha256=1heBmPgB-W5Zf9gwViM7NrqvHpZlVYeMN7jpY8Qkytc,5479
+torchx/workspace/api.py,sha256=PtDkGTC5lX03pRoYpuMz2KCmM1ZOycRP1UknqvNb97Y,6341
 torchx/workspace/dir_workspace.py,sha256=npNW_IjUZm_yS5r-8hrRkH46ndDd9a_eApT64m1S1T4,2268
 torchx/workspace/docker_workspace.py,sha256=PFu2KQNVC-0p2aKJ-W_BKA9ZOmXdCY2ABEkCExp3udQ,10269
-torchx_nightly-2024.10.15.dist-info/LICENSE,sha256=WVHfXhFC0Ia8LTKt_nJVYobdqTJVg_4J3Crrfm2A8KQ,1721
-torchx_nightly-2024.10.15.dist-info/METADATA,sha256=1HVUgbu59EwuBqagzVBiQzHHTCBO7yaJ8ZAvvXUbMrU,6170
-torchx_nightly-2024.10.15.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-torchx_nightly-2024.10.15.dist-info/entry_points.txt,sha256=T328AMXeKI3JZnnxfkEew2ZcMN1oQDtkXjMz7lkV-P4,169
-torchx_nightly-2024.10.15.dist-info/top_level.txt,sha256=pxew3bc2gsiViS0zADs0jb6kC5v8o_Yy_85fhHj_J1A,7
-torchx_nightly-2024.10.15.dist-info/RECORD,,
+torchx_nightly-2024.10.17.dist-info/LICENSE,sha256=WVHfXhFC0Ia8LTKt_nJVYobdqTJVg_4J3Crrfm2A8KQ,1721
+torchx_nightly-2024.10.17.dist-info/METADATA,sha256=Zbb7ZcE4TadqUN70RDvZETyDGd6a9QW7wp8Ho2BHFKs,6170
+torchx_nightly-2024.10.17.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+torchx_nightly-2024.10.17.dist-info/entry_points.txt,sha256=T328AMXeKI3JZnnxfkEew2ZcMN1oQDtkXjMz7lkV-P4,169
+torchx_nightly-2024.10.17.dist-info/top_level.txt,sha256=pxew3bc2gsiViS0zADs0jb6kC5v8o_Yy_85fhHj_J1A,7
+torchx_nightly-2024.10.17.dist-info/RECORD,,