nvidia-nat 1.1.0a20251020__py3-none-any.whl

nat/utils/decorators.py

@@ -0,0 +1,210 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Deprecation utilities.
+
+This module provides helpers to standardize deprecation signaling across the
+codebase:
+
+- ``issue_deprecation_warning``: Builds and emits a single deprecation message
+  per function using the standard logging pipeline.
+- ``deprecated``: A decorator that wraps sync/async functions and generators to
+  log a one-time deprecation message upon first use. It supports optional
+  metadata, a planned removal version, a suggested replacement, and an
+  optional feature name label.
+
+Messages are emitted via ``logging.getLogger(__name__).warning`` (not
+``warnings.warn``) so they appear in normal application logs and respect global
+logging configuration. Each unique function logs at most once per process.
+"""
+
+import functools
+import inspect
+import logging
+from collections.abc import AsyncGenerator
+from collections.abc import Callable
+from collections.abc import Generator
+from typing import Any
+from typing import TypeVar
+from typing import overload
+
+logger = logging.getLogger(__name__)
+
+_warning_issued = set()
+
+# Type variables for overloads
+F = TypeVar('F', bound=Callable[..., Any])
+
+
+def issue_deprecation_warning(function_name: str,
+                              removal_version: str | None = None,
+                              replacement: str | None = None,
+                              reason: str | None = None,
+                              feature_name: str | None = None,
+                              metadata: dict[str, Any] | None = None) -> None:
+    """
+    Log a deprecation warning message for the function.
+
+    A warning is emitted only once per function. When a ``metadata`` dict
+    is supplied, it is appended to the log entry to provide extra context
+    (e.g., version, author, feature flag).
+
+    Args:
+        function_name: The name of the deprecated function
+        removal_version: The version when the function will be removed
+        replacement: What to use instead of this function
+        reason: Why the function is being deprecated
+        feature_name: Optional name of the feature that is deprecated
+        metadata: Optional dictionary of metadata to log with the warning
+    """
+    if function_name not in _warning_issued:
+        # Build the deprecation message
+        if feature_name:
+            warning_message = f"{feature_name} is deprecated"
+        else:
+            warning_message = f"Function {function_name} is deprecated"
+
+        if removal_version:
+            warning_message += f" and will be removed in version {removal_version}"
+        else:
+            warning_message += " and will be removed in a future release"
+
+        warning_message += "."
+
+        if reason:
+            warning_message += f" Reason: {reason}."
+
+        if replacement:
+            warning_message += f" Use '{replacement}' instead."
+
+        if metadata:
+            warning_message += f" | Metadata: {metadata}"
+
+        # Issue warning and save function name to avoid duplicate warnings
+        logger.warning(warning_message)
+        _warning_issued.add(function_name)
+
+
+# Overloads for different function types
+@overload
+def deprecated(func: F,
+               *,
+               removal_version: str | None = None,
+               replacement: str | None = None,
+               reason: str | None = None,
+               feature_name: str | None = None,
+               metadata: dict[str, Any] | None = None) -> F:
+    """Overload for direct decorator usage (when called without parentheses)."""
+    ...
+
+
+@overload
+def deprecated(*,
+               removal_version: str | None = None,
+               replacement: str | None = None,
+               reason: str | None = None,
+               feature_name: str | None = None,
+               metadata: dict[str, Any] | None = None) -> Callable[[F], F]:
+    """Overload for decorator factory usage (when called with parentheses)."""
+    ...
+
+
+def deprecated(func: Any = None,
+               *,
+               removal_version: str | None = None,
+               replacement: str | None = None,
+               reason: str | None = None,
+               feature_name: str | None = None,
+               metadata: dict[str, Any] | None = None) -> Any:
+    """
+    Decorator that can wrap any type of function (sync, async, generator,
+    async generator) and logs a deprecation warning.
+
+    Args:
+        func: The function to be decorated.
+        removal_version: The version when the function will be removed
+        replacement: What to use instead of this function
+        reason: Why the function is being deprecated
+        feature_name: Optional name of the feature that is deprecated. If provided, the warning will be
+        prefixed with "The <feature_name> feature is deprecated".
+        metadata: Optional dictionary of metadata to log with the warning. This can include information
+        like version, author, etc. If provided, the metadata will be
+        logged alongside the deprecation warning.
+    """
+    function_name: str = f"{func.__module__}.{func.__qualname__}" if func else "<unknown_function>"
+
+    # If called as @deprecated(...) but not immediately passed a function
+    if func is None:
+
+        def decorator_wrapper(actual_func):
+            return deprecated(actual_func,
+                              removal_version=removal_version,
+                              replacement=replacement,
+                              reason=reason,
+                              feature_name=feature_name,
+                              metadata=metadata)
+
+        return decorator_wrapper
+
+    # --- Validate metadata ---
+    if metadata is not None:
+        if not isinstance(metadata, dict):
+            raise TypeError("metadata must be a dict[str, Any].")
+        if any(not isinstance(k, str) for k in metadata.keys()):
+            raise TypeError("All metadata keys must be strings.")
+
+    # --- Now detect the function type and wrap accordingly ---
+    if inspect.isasyncgenfunction(func):
+        # ---------------------
+        # ASYNC GENERATOR
+        # ---------------------
+
+        @functools.wraps(func)
+        async def async_gen_wrapper(*args, **kwargs) -> AsyncGenerator[Any, Any]:
+            issue_deprecation_warning(function_name, removal_version, replacement, reason, feature_name, metadata)
+            async for item in func(*args, **kwargs):
+                yield item  # yield the original item
+
+        return async_gen_wrapper
+
+    if inspect.iscoroutinefunction(func):
+        # ---------------------
+        # ASYNC FUNCTION
+        # ---------------------
+        @functools.wraps(func)
+        async def async_wrapper(*args, **kwargs) -> Any:
+            issue_deprecation_warning(function_name, removal_version, replacement, reason, feature_name, metadata)
+            result = await func(*args, **kwargs)
+            return result
+
+        return async_wrapper
+
+    if inspect.isgeneratorfunction(func):
+        # ---------------------
+        # SYNC GENERATOR
+        # ---------------------
+        @functools.wraps(func)
+        def sync_gen_wrapper(*args, **kwargs) -> Generator[Any, Any, Any]:
+            issue_deprecation_warning(function_name, removal_version, replacement, reason, feature_name, metadata)
+            yield from func(*args, **kwargs)  # yield the original item
+
+        return sync_gen_wrapper
+
+    @functools.wraps(func)
+    def sync_wrapper(*args, **kwargs) -> Any:
+        issue_deprecation_warning(function_name, removal_version, replacement, reason, feature_name, metadata)
+        result = func(*args, **kwargs)
+        return result
+
+    return sync_wrapper

nat/utils/dump_distro_mapping.py

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+
+from nat.runtime.loader import get_all_entrypoints_distro_mapping
+
+
+def dump_distro_mapping(path: str):
+    mapping = get_all_entrypoints_distro_mapping()
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(mapping, f, indent=4)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--path", type=str, required=True)
+    args = parser.parse_args()
+    dump_distro_mapping(args.path)

nat/utils/exception_handlers/automatic_retries.py

@@ -0,0 +1,342 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import asyncio
+import copy
+import functools
+import inspect
+import logging
+import re
+import time
+import types
+from collections.abc import Callable
+from collections.abc import Iterable
+from collections.abc import Sequence
+from typing import Any
+from typing import TypeVar
+
+T = TypeVar("T")
+Exc = tuple[type[BaseException], ...]  # exception classes
+CodePattern = int | str | range  # for retry_codes argument
+logger = logging.getLogger(__name__)
+
+# ──────────────────────────────────────────────────────────────────────────────
+#  Helpers: status-code extraction & pattern matching
+# ──────────────────────────────────────────────────────────────────────────────
+_CODE_ATTRS = ("code", "status", "status_code", "http_status")
+
+
+def _extract_status_code(exc: BaseException) -> int | None:
+    """Return a numeric status code found inside *exc*, else None."""
+    for attr in _CODE_ATTRS:
+        if hasattr(exc, attr):
+            try:
+                return int(getattr(exc, attr))
+            except (TypeError, ValueError):
+                pass
+    if exc.args:
+        try:
+            return int(exc.args[0])
+        except (TypeError, ValueError):
+            pass
+    return None
+
+
+def _pattern_to_regex(pat: str) -> re.Pattern[str]:
+    """
+    Convert simple wildcard pattern (“4xx”, “5*”, “40x”) to a ^regex$.
+    Rule:  ‘x’ or ‘*’ ⇒ any digit.
+    """
+    escaped = re.escape(pat)
+    return re.compile("^" + escaped.replace(r"\*", r"\d").replace("x", r"\d") + "$")
+
+
+def _code_matches(code: int, pat: CodePattern) -> bool:
+    if isinstance(pat, int):
+        return code == pat
+    if isinstance(pat, range):
+        return code in pat
+    return bool(_pattern_to_regex(pat).match(str(code)))
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+#  Unified retry-decision helper
+# ──────────────────────────────────────────────────────────────────────────────
+def _want_retry(
+    exc: BaseException,
+    *,
+    code_patterns: Sequence[CodePattern] | None,
+    msg_substrings: Sequence[str] | None,
+) -> bool:
+    """
+    Return True if the exception satisfies *either* (when provided):
+       • code_patterns  – matches status-code pattern(s)
+       • msg_substrings – contains any of the substrings (case-insensitive)
+    """
+
+    if not code_patterns and not msg_substrings:
+        logger.info("Retrying on exception %s without extra filters", exc)
+        return True
+
+    # -------- status-code filter --------
+    if code_patterns is not None:
+        code = _extract_status_code(exc)
+        if any(_code_matches(code, p) for p in code_patterns):
+            logger.info("Retrying on exception %s with matched code %s", exc, code)
+            return True
+
+    # -------- message filter -----------
+    if msg_substrings is not None:
+        msg = str(exc).lower()
+        if any(s.lower() in msg for s in msg_substrings):
+            logger.info("Retrying on exception %s with matched message %s", exc, msg)
+            return True
+
+    return False
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+#  Core decorator factory (sync / async / (a)gen)
+# ──────────────────────────────────────────────────────────────────────────────
+def _retry_decorator(
+    *,
+    retries: int = 3,
+    base_delay: float = 0.25,
+    backoff: float = 2.0,
+    retry_on: Exc = (Exception, ),
+    retry_codes: Sequence[CodePattern] | None = None,
+    retry_on_messages: Sequence[str] | None = None,
+    deepcopy: bool = False,
+    instance_context_aware: bool = False,
+) -> Callable[[Callable[..., T]], Callable[..., T]]:
+    """
+    Build a decorator that retries with exponential back-off *iff*:
+
+      • the raised exception is an instance of one of `retry_on`
+      • AND `_want_retry()` returns True (i.e. matches codes/messages filters)
+
+    If both `retry_codes` and `retry_on_messages` are None, all exceptions are retried.
+
+    deepcopy:
+        If True, each retry receives deep‑copied *args and **kwargs* to avoid
+        mutating shared state between attempts.
+
+    instance_context_aware:
+        If True, the decorator will check for a retry context flag on the first
+        argument (assumed to be 'self'). If the flag is set, retries are skipped
+        to prevent retry storms in nested method calls.
+    """
+
+    def decorate(fn: Callable[..., T]) -> Callable[..., T]:
+        use_deepcopy = deepcopy
+        use_context_aware = instance_context_aware
+
+        class _RetryContext:
+            """Context manager for instance-level retry gating."""
+
+            __slots__ = ("_obj", "_enabled", "_active")
+
+            def __init__(self, args: tuple[Any, ...]):
+                self._obj = args[0] if (use_context_aware and args) else None
+                self._enabled = bool(self._obj)
+                self._active = False
+
+            def __enter__(self):
+                if not self._enabled:
+                    return False
+                try:
+                    # If already in retry context, signal caller to skip retries
+                    if getattr(self._obj, "_in_retry_context", False):
+                        return True
+                    object.__setattr__(self._obj, "_in_retry_context", True)
+                    self._active = True
+                    return False
+                except Exception:
+                    # If we cannot set the attribute, behave as if context isn't enabled
+                    self._enabled = False
+                    return False
+
+            def __exit__(self, _exc_type, _exc, _tb):
+                if self._enabled and self._active:
+                    try:
+                        object.__setattr__(self._obj, "_in_retry_context", False)
+                    except Exception:
+                        pass
+
+        async def _call_with_retry_async(*args, **kw) -> T:
+            with _RetryContext(args) as already_in_context:
+                if already_in_context:
+                    return await fn(*args, **kw)
+                delay = base_delay
+                for attempt in range(retries):
+                    call_args = copy.deepcopy(args) if use_deepcopy else args
+                    call_kwargs = copy.deepcopy(kw) if use_deepcopy else kw
+                    try:
+                        return await fn(*call_args, **call_kwargs)
+                    except retry_on as exc:
+                        if (not _want_retry(exc, code_patterns=retry_codes, msg_substrings=retry_on_messages)
+                                or attempt == retries - 1):
+                            raise
+                        await asyncio.sleep(delay)
+                        delay *= backoff
+
+        async def _agen_with_retry(*args, **kw):
+            with _RetryContext(args) as already_in_context:
+                if already_in_context:
+                    async for item in fn(*args, **kw):
+                        yield item
+                    return
+                delay = base_delay
+                for attempt in range(retries):
+                    call_args = copy.deepcopy(args) if use_deepcopy else args
+                    call_kwargs = copy.deepcopy(kw) if use_deepcopy else kw
+                    try:
+                        async for item in fn(*call_args, **call_kwargs):
+                            yield item
+                        return
+                    except retry_on as exc:
+                        if (not _want_retry(exc, code_patterns=retry_codes, msg_substrings=retry_on_messages)
+                                or attempt == retries - 1):
+                            raise
+                        await asyncio.sleep(delay)
+                        delay *= backoff
+
+        def _gen_with_retry(*args, **kw) -> Iterable[Any]:
+            with _RetryContext(args) as already_in_context:
+                if already_in_context:
+                    yield from fn(*args, **kw)
+                    return
+                delay = base_delay
+                for attempt in range(retries):
+                    call_args = copy.deepcopy(args) if use_deepcopy else args
+                    call_kwargs = copy.deepcopy(kw) if use_deepcopy else kw
+                    try:
+                        yield from fn(*call_args, **call_kwargs)
+                        return
+                    except retry_on as exc:
+                        if (not _want_retry(exc, code_patterns=retry_codes, msg_substrings=retry_on_messages)
+                                or attempt == retries - 1):
+                            raise
+                        time.sleep(delay)
+                        delay *= backoff
+
+        def _sync_with_retry(*args, **kw) -> T:
+            with _RetryContext(args) as already_in_context:
+                if already_in_context:
+                    return fn(*args, **kw)
+                delay = base_delay
+                for attempt in range(retries):
+                    call_args = copy.deepcopy(args) if use_deepcopy else args
+                    call_kwargs = copy.deepcopy(kw) if use_deepcopy else kw
+                    try:
+                        return fn(*call_args, **call_kwargs)
+                    except retry_on as exc:
+                        if (not _want_retry(exc, code_patterns=retry_codes, msg_substrings=retry_on_messages)
+                                or attempt == retries - 1):
+                            raise
+                        time.sleep(delay)
+                        delay *= backoff
+
+        # Decide which wrapper to return
+        if inspect.iscoroutinefunction(fn):
+            wrapper = _call_with_retry_async
+        elif inspect.isasyncgenfunction(fn):
+            wrapper = _agen_with_retry
+        elif inspect.isgeneratorfunction(fn):
+            wrapper = _gen_with_retry
+        else:
+            wrapper = _sync_with_retry
+
+        return functools.wraps(fn)(wrapper)  # type: ignore[return-value]
+
+    return decorate
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+#  Public helper : patch_with_retry
+# ──────────────────────────────────────────────────────────────────────────────
+def patch_with_retry(
+    obj: Any,
+    *,
+    retries: int = 3,
+    base_delay: float = 0.25,
+    backoff: float = 2.0,
+    retry_on: Exc = (Exception, ),
+    retry_codes: Sequence[CodePattern] | None = None,
+    retry_on_messages: Sequence[str] | None = None,
+    deepcopy: bool = False,
+) -> Any:
+    """
+    Patch *obj* instance-locally so **every public method** retries on failure.
+
+    Extra filters
+    -------------
+    retry_codes
+        Same as before – ints, ranges, or wildcard strings (“4xx”, “5*”…).
+    retry_on_messages
+        List of *substring* patterns.  We retry only if **any** pattern
+        appears (case-insensitive) in `str(exc)`.
+    deepcopy:
+        If True, each retry receives deep‑copied *args and **kwargs* to avoid
+        mutating shared state between attempts.
+    """
+    deco = _retry_decorator(
+        retries=retries,
+        base_delay=base_delay,
+        backoff=backoff,
+        retry_on=retry_on,
+        retry_codes=retry_codes,
+        retry_on_messages=retry_on_messages,
+        deepcopy=deepcopy,
+        instance_context_aware=True,  # Prevent retry storms
+    )
+
+    # Choose attribute source: the *class* to avoid triggering __getattr__
+    cls = obj if inspect.isclass(obj) else type(obj)
+    cls_name = getattr(cls, "__name__", str(cls))
+
+    for name, _ in inspect.getmembers(cls, callable):
+        descriptor = inspect.getattr_static(cls, name)
+
+        # Skip dunders, privates and all descriptors we must not wrap
+        if (name.startswith("_") or isinstance(descriptor, property | staticmethod | classmethod)):
+            continue
+
+        original = descriptor.__func__ if isinstance(descriptor, types.MethodType) else descriptor
+        wrapped = deco(original)
+
+        try:  # instance‑level first
+            if not inspect.isclass(obj):
+                object.__setattr__(obj, name, types.MethodType(wrapped, obj))
+                continue
+        except Exception as exc:
+            logger.info(
+                "Instance‑level patch failed for %s.%s (%s); "
+                "falling back to class‑level patch.",
+                cls_name,
+                name,
+                exc,
+            )
+
+        try:  # class‑level fallback
+            setattr(cls, name, wrapped)
+        except Exception as exc:
+            logger.info(
+                "Cannot patch method %s.%s with automatic retries: %s",
+                cls_name,
+                name,
+                exc,
+            )
+
+    return obj