torchx-nightly 2025.9.10__py3-none-any.whl → 2025.9.11__py3-none-any.whl

torchx/specs/file_linter.py

@@ -11,8 +11,9 @@ import abc
 import argparse
 import ast
 import inspect
+import sys
 from dataclasses import dataclass
-from typing import Callable, cast, Dict, List, Optional, Tuple
+from typing import Callable, Dict, List, Optional, Tuple
 
 from docstring_parser import parse
 from torchx.util.io import read_conf_file
@@ -98,7 +99,7 @@ class LinterMessage:
     severity: str = "error"
 
 
-class TorchxFunctionValidator(abc.ABC):
+class ComponentFunctionValidator(abc.ABC):
     @abc.abstractmethod
     def validate(self, app_specs_func_def: ast.FunctionDef) -> List[LinterMessage]:
         """
@@ -116,7 +117,55 @@ class TorchxFunctionValidator(abc.ABC):
         )
 
 
-class TorchxFunctionArgsValidator(TorchxFunctionValidator):
+def OK() -> list[LinterMessage]:
+    return []  # empty linter error means validation passes
+
+
+def is_primitive(arg: ast.expr) -> bool:
+    # whether the arg is a primitive type (e.g. int, float, str, bool)
+    return isinstance(arg, ast.Name)
+
+
+def get_generic_type(arg: ast.expr) -> ast.expr:
+    # returns the slice expr of a subscripted type
+    # `arg` must be an instance of ast.Subscript (caller checks)
+    # in this validator's context, this is the generic type of a container type
+    # e.g. for Optional[str] returns the expr for str
+
+    assert isinstance(arg, ast.Subscript)  # e.g. arg = C[T]
+
+    if isinstance(arg.slice, ast.Index):  # python>=3.10
+        return arg.slice.value
+    else:  # python-3.9
+        return arg.slice
+
+
+def get_optional_type(arg: ast.expr) -> Optional[ast.expr]:
+    """
+    Returns the type parameter ``T`` of ``Optional[T]`` or ``None`` if `arg``
+    is not an ``Optional``. Handles both:
+        1. ``typing.Optional[T]`` (python<3.10)
+        2.  ``T | None`` or ``None | T`` (python>=3.10 - PEP 604)
+    """
+    # case 1: 'a: Optional[T]'
+    if isinstance(arg, ast.Subscript) and arg.value.id == "Optional":
+        return get_generic_type(arg)
+
+    # case 2: 'a: T | None' or 'a: None | T'
+    if sys.version_info >= (3, 10):  # PEP 604 introduced in python-3.10
+        if isinstance(arg, ast.BinOp) and isinstance(arg.op, ast.BitOr):
+            if isinstance(arg.right, ast.Constant) and arg.right.value is None:
+                return arg.left
+            if isinstance(arg.left, ast.Constant) and arg.left.value is None:
+                return arg.right
+
+    # case 3: is not optional
+    return None
+
+
+class ArgTypeValidator(ComponentFunctionValidator):
+    """Validates component function's argument types."""
+
     def validate(self, app_specs_func_def: ast.FunctionDef) -> List[LinterMessage]:
         linter_errors = []
         for arg_def in app_specs_func_def.args.args:
@@ -133,53 +182,68 @@ class TorchxFunctionArgsValidator(TorchxFunctionValidator):
         return linter_errors
 
     def _validate_arg_def(
-        self, function_name: str, arg_def: ast.arg
+        self, function_name: str, arg: ast.arg
     ) -> List[LinterMessage]:
-        if not arg_def.annotation:
-            return [
-                self._gen_linter_message(
-                    f"Arg {arg_def.arg} missing type annotation", arg_def.lineno
-                )
-            ]
-        if isinstance(arg_def.annotation, ast.Name):
+        arg_type = arg.annotation  # type hint
+
+        def ok() -> list[LinterMessage]:
+            # return value when validation passes (e.g. no linter errors)
             return []
-        complex_type_def = cast(ast.Subscript, none_throws(arg_def.annotation))
-        if complex_type_def.value.id == "Optional":
-            # ast module in python3.9 does not have ast.Index wrapper
-            if isinstance(complex_type_def.slice, ast.Index):
-                complex_type_def = complex_type_def.slice.value
-            else:
-                complex_type_def = complex_type_def.slice
-            # Check if type is Optional[primitive_type]
-            if isinstance(complex_type_def, ast.Name):
-                return []
-        # Check if type is Union[Dict,List]
-        type_name = complex_type_def.value.id
-        if type_name != "Dict" and type_name != "List":
-            desc = (
-                f"`{function_name}` allows only Dict, List as complex types."
-                f"Argument `{arg_def.arg}` has: {type_name}"
-            )
-            return [self._gen_linter_message(desc, arg_def.lineno)]
-        linter_errors = []
-        # ast module in python3.9 does not have objects wrapped in ast.Index
-        if isinstance(complex_type_def.slice, ast.Index):
-            sub_type = complex_type_def.slice.value
+
+        def err(reason: str) -> list[LinterMessage]:
+            msg = f"{reason} for argument {ast.unparse(arg)!r} in function {function_name!r}"
+            return [self._gen_linter_message(msg, arg.lineno)]
+
+        if not arg_type:
+            return err("Missing type annotation")
+
+        # Case 1: optional
+        if T := get_optional_type(arg_type):
+            # NOTE: optional types can be primitives or any of the allowed container types
+            #   so check if arg is an optional, and if so, run the rest of the validation with the unpacked type
+            arg_type = T
+
+        # Case 2: int, float, str, bool
+        if is_primitive(arg_type):
+            return ok()
+        # Case 3: Containers (Dict, List, Tuple)
+        elif isinstance(arg_type, ast.Subscript):
+            container_type = arg_type.value.id
+
+            if container_type in ["Dict", "dict"]:
+                KV = get_generic_type(arg_type)
+
+                assert isinstance(KV, ast.Tuple)  # dict[K,V] has ast.Tuple slice
+
+                K, V = KV.elts
+                if not is_primitive(K):
+                    return err(f"Non-primitive key type {ast.unparse(K)!r}")
+                if not is_primitive(V):
+                    return err(f"Non-primitive value type {ast.unparse(V)!r}")
+                return ok()
+            elif container_type in ["List", "list"]:
+                T = get_generic_type(arg_type)
+                if is_primitive(T):
+                    return ok()
+                else:
+                    return err(f"Non-primitive element type {ast.unparse(T)!r}")
+            elif container_type in ["Tuple", "tuple"]:
+                E_N = get_generic_type(arg_type)
+                assert isinstance(E_N, ast.Tuple)  # tuple[...] has ast.Tuple slice
+
+                for e in E_N.elts:
+                    if not is_primitive(e):
+                        return err(f"Non-primitive element type '{ast.unparse(e)!r}'")
+
+                return ok()
+
+            return err(f"Unsupported container type {container_type!r}")
         else:
-            sub_type = complex_type_def.slice
-        if type_name == "Dict":
-            sub_type_tuple = cast(ast.Tuple, sub_type)
-            for el in sub_type_tuple.elts:
-                if not isinstance(el, ast.Name):
-                    desc = "Dict can only have primitive types"
-                    linter_errors.append(self._gen_linter_message(desc, arg_def.lineno))
-        elif not isinstance(sub_type, ast.Name):
-            desc = "List can only have primitive types"
-            linter_errors.append(self._gen_linter_message(desc, arg_def.lineno))
-        return linter_errors
+            return err(f"Unsupported argument type {ast.unparse(arg_type)!r}")
 
 
-class TorchxReturnValidator(TorchxFunctionValidator):
+class ReturnTypeValidator(ComponentFunctionValidator):
+    """Validates that component functions always return AppDef type"""
 
     def __init__(self, supported_return_type: str) -> None:
         super().__init__()
@@ -231,7 +295,7 @@ class TorchxReturnValidator(TorchxFunctionValidator):
         return linter_errors
 
 
-class TorchFunctionVisitor(ast.NodeVisitor):
+class ComponentFnVisitor(ast.NodeVisitor):
     """
     Visitor that finds the component_function and runs registered validators on it.
     Current registered validators:
@@ -252,12 +316,12 @@ class TorchFunctionVisitor(ast.NodeVisitor):
     def __init__(
         self,
         component_function_name: str,
-        validators: Optional[List[TorchxFunctionValidator]],
+        validators: Optional[List[ComponentFunctionValidator]],
     ) -> None:
         if validators is None:
-            self.validators: List[TorchxFunctionValidator] = [
-                TorchxFunctionArgsValidator(),
-                TorchxReturnValidator("AppDef"),
+            self.validators: List[ComponentFunctionValidator] = [
+                ArgTypeValidator(),
+                ReturnTypeValidator("AppDef"),
             ]
         else:
             self.validators = validators
@@ -279,7 +343,7 @@ class TorchFunctionVisitor(ast.NodeVisitor):
 def validate(
     path: str,
     component_function: str,
-    validators: Optional[List[TorchxFunctionValidator]],
+    validators: Optional[List[ComponentFunctionValidator]] = None,
 ) -> List[LinterMessage]:
     """
     Validates the function to make sure it complies the component standard.
@@ -309,7 +373,7 @@ def validate(
             severity="error",
         )
         return [linter_message]
-    visitor = TorchFunctionVisitor(component_function, validators)
+    visitor = ComponentFnVisitor(component_function, validators)
     visitor.visit(module)
     linter_errors = visitor.linter_errors
     if not visitor.visited_function:

torchx/specs/finder.py

@@ -19,7 +19,11 @@ from pathlib import Path
 from types import ModuleType
 from typing import Any, Callable, Dict, Generator, List, Optional, Union
 
-from torchx.specs.file_linter import get_fn_docstring, TorchxFunctionValidator, validate
+from torchx.specs.file_linter import (
+    ComponentFunctionValidator,
+    get_fn_docstring,
+    validate,
+)
 from torchx.util import entrypoints
 from torchx.util.io import read_conf_file
 from torchx.util.types import none_throws
@@ -64,7 +68,7 @@ class _Component:
 class ComponentsFinder(abc.ABC):
     @abc.abstractmethod
     def find(
-        self, validators: Optional[List[TorchxFunctionValidator]]
+        self, validators: Optional[List[ComponentFunctionValidator]]
     ) -> List[_Component]:
         """
         Retrieves a set of components. A component is defined as a python
@@ -210,7 +214,7 @@ class ModuleComponentsFinder(ComponentsFinder):
                     yield self._try_import(module_info.name)
 
     def find(
-        self, validators: Optional[List[TorchxFunctionValidator]]
+        self, validators: Optional[List[ComponentFunctionValidator]]
     ) -> List[_Component]:
         components = []
         for m in self._iter_modules_recursive(self.base_module):
@@ -230,7 +234,7 @@ class ModuleComponentsFinder(ComponentsFinder):
             return module
 
     def _get_components_from_module(
-        self, module: ModuleType, validators: Optional[List[TorchxFunctionValidator]]
+        self, module: ModuleType, validators: Optional[List[ComponentFunctionValidator]]
     ) -> List[_Component]:
         functions = getmembers(module, isfunction)
         component_defs = []
@@ -269,7 +273,7 @@ class CustomComponentsFinder(ComponentsFinder):
         self,
         path: str,
         function_name: str,
-        validators: Optional[List[TorchxFunctionValidator]],
+        validators: Optional[List[ComponentFunctionValidator]],
     ) -> List[str]:
         linter_errors = validate(path, function_name, validators)
         return [linter_error.description for linter_error in linter_errors]
@@ -283,13 +287,15 @@ class CustomComponentsFinder(ComponentsFinder):
         my_component defined in some_file.py, imported in other_file.py
         and the component is invoked as other_file.py:my_component
         """
-        path_to_function_decl = inspect.getabsfile(function)
+        # Unwrap decorated functions to get the original function
+        unwrapped_function = inspect.unwrap(function)
+        path_to_function_decl = inspect.getabsfile(unwrapped_function)
         if path_to_function_decl is None or not os.path.isfile(path_to_function_decl):
             return self._filepath
         return path_to_function_decl
 
     def find(
-        self, validators: Optional[List[TorchxFunctionValidator]]
+        self, validators: Optional[List[ComponentFunctionValidator]]
     ) -> List[_Component]:
 
         file_source = read_conf_file(self._filepath)
@@ -321,7 +327,7 @@ class CustomComponentsFinder(ComponentsFinder):
 
 
 def _load_custom_components(
-    validators: Optional[List[TorchxFunctionValidator]],
+    validators: Optional[List[ComponentFunctionValidator]],
 ) -> List[_Component]:
     component_modules = {
         name: load_fn()
@@ -346,7 +352,7 @@ def _load_custom_components(
 
 
 def _load_components(
-    validators: Optional[List[TorchxFunctionValidator]],
+    validators: Optional[List[ComponentFunctionValidator]],
 ) -> Dict[str, _Component]:
     """
     Loads either the custom component defs from the entrypoint ``[torchx.components]``
@@ -368,7 +374,7 @@ _components: Optional[Dict[str, _Component]] = None
 
 
 def _find_components(
-    validators: Optional[List[TorchxFunctionValidator]],
+    validators: Optional[List[ComponentFunctionValidator]],
 ) -> Dict[str, _Component]:
     global _components
     if not _components:
@@ -381,7 +387,7 @@ def _is_custom_component(component_name: str) -> bool:
 
 
 def _find_custom_components(
-    name: str, validators: Optional[List[TorchxFunctionValidator]]
+    name: str, validators: Optional[List[ComponentFunctionValidator]]
 ) -> Dict[str, _Component]:
     if ":" not in name:
         raise ValueError(
@@ -393,7 +399,7 @@ def _find_custom_components(
 
 
 def get_components(
-    validators: Optional[List[TorchxFunctionValidator]] = None,
+    validators: Optional[List[ComponentFunctionValidator]] = None,
 ) -> Dict[str, _Component]:
     """
     Returns all custom components registered via ``[torchx.components]`` entrypoints
@@ -448,7 +454,7 @@ def get_components(
 
 
 def get_component(
-    name: str, validators: Optional[List[TorchxFunctionValidator]] = None
+    name: str, validators: Optional[List[ComponentFunctionValidator]] = None
 ) -> _Component:
     """
     Retrieves components by the provided name.
@@ -477,7 +483,7 @@ def get_component(
 
 
 def get_builtin_source(
-    name: str, validators: Optional[List[TorchxFunctionValidator]] = None
+    name: str, validators: Optional[List[ComponentFunctionValidator]] = None
 ) -> str:
     """
     Returns a string of the the builtin component's function source code

{torchx_nightly-2025.9.10.dist-info → torchx_nightly-2025.9.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: torchx-nightly
-Version: 2025.9.10
+Version: 2025.9.11
 Summary: TorchX SDK and Components
 Home-page: https://github.com/pytorch/torchx
 Author: TorchX Devs

{torchx_nightly-2025.9.10.dist-info → torchx_nightly-2025.9.11.dist-info}/RECORD

@@ -85,8 +85,8 @@ torchx/schedulers/ray/ray_driver.py,sha256=RdaCLfth16ky-5PDVOWRe_RuheWJu9xufWux2
 torchx/specs/__init__.py,sha256=Gw_2actqR_oWFtxEkGXCxGk_yrWK5JDZzwysyyqmXao,6438
 torchx/specs/api.py,sha256=wkhHOxeWH_tFO3npKqPhNg4VX2NH5gPIFEylkPBo3AU,41315
 torchx/specs/builders.py,sha256=aozVl4q3h0mY5DDJCY1M1CyLC9SW66KJy8JIih8bZJo,13810
-torchx/specs/file_linter.py,sha256=QCwob5STTBuy8RsxaevTI-Dk6R8siDJn81LyaOwazes,12333
-torchx/specs/finder.py,sha256=lMCS1hUR-x7j4sAjfJZnKFeS1RlkFTeJtit4EL_ErIo,18182
+torchx/specs/file_linter.py,sha256=6_aoeuS5d9UwXseKKfPgWNTwxj-f7G1i3uO9mQepti4,14402
+torchx/specs/finder.py,sha256=8peKWCKVutCRP0WmffRqsZNqjIh6xfxMKbVA4aEPM_M,18368
 torchx/specs/named_resources_aws.py,sha256=ISjHtifRJqB8u7PeAMiyLyO_S0WCaZiK-CFF3qe6JDU,11415
 torchx/specs/named_resources_generic.py,sha256=Sg4tAdqiiWDrDz2Lj_pnfsjzGIXKTou73wPseh6j55w,2646
 torchx/specs/test/components/__init__.py,sha256=J8qjUOysmcMAek2KFN13mViOXZxTYc5vCrF02t3VuFU,223
@@ -115,9 +115,9 @@ torchx/workspace/__init__.py,sha256=FqN8AN4VhR1C_SBY10MggQvNZmyanbbuPuE-JCjkyUY,
 torchx/workspace/api.py,sha256=PtDkGTC5lX03pRoYpuMz2KCmM1ZOycRP1UknqvNb97Y,6341
 torchx/workspace/dir_workspace.py,sha256=npNW_IjUZm_yS5r-8hrRkH46ndDd9a_eApT64m1S1T4,2268
 torchx/workspace/docker_workspace.py,sha256=PFu2KQNVC-0p2aKJ-W_BKA9ZOmXdCY2ABEkCExp3udQ,10269
-torchx_nightly-2025.9.10.dist-info/LICENSE,sha256=WVHfXhFC0Ia8LTKt_nJVYobdqTJVg_4J3Crrfm2A8KQ,1721
-torchx_nightly-2025.9.10.dist-info/METADATA,sha256=LbbQDhoaKCVyMGIi5GMTr2O7hI-kSSSF894v1eYiL3I,6104
-torchx_nightly-2025.9.10.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-torchx_nightly-2025.9.10.dist-info/entry_points.txt,sha256=T328AMXeKI3JZnnxfkEew2ZcMN1oQDtkXjMz7lkV-P4,169
-torchx_nightly-2025.9.10.dist-info/top_level.txt,sha256=pxew3bc2gsiViS0zADs0jb6kC5v8o_Yy_85fhHj_J1A,7
-torchx_nightly-2025.9.10.dist-info/RECORD,,
+torchx_nightly-2025.9.11.dist-info/LICENSE,sha256=WVHfXhFC0Ia8LTKt_nJVYobdqTJVg_4J3Crrfm2A8KQ,1721
+torchx_nightly-2025.9.11.dist-info/METADATA,sha256=38npZBvhuRMn5mHdg_tuQNWP3Mp0xEWaDdpRbV2YE58,6104
+torchx_nightly-2025.9.11.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+torchx_nightly-2025.9.11.dist-info/entry_points.txt,sha256=T328AMXeKI3JZnnxfkEew2ZcMN1oQDtkXjMz7lkV-P4,169
+torchx_nightly-2025.9.11.dist-info/top_level.txt,sha256=pxew3bc2gsiViS0zADs0jb6kC5v8o_Yy_85fhHj_J1A,7
+torchx_nightly-2025.9.11.dist-info/RECORD,,