nv-ingest-api 26.1.0rc4__py3-none-any.whl

nv_ingest_api/util/imports/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0

nv_ingest_api/util/imports/callable_signatures.py

@@ -0,0 +1,108 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+
+import inspect
+from pydantic import BaseModel
+
+from nv_ingest_api.internal.primitives.ingest_control_message import IngestControlMessage
+
+
+def ingest_stage_callable_signature(sig: inspect.Signature):
+    """
+    Validates that a callable has the signature:
+        (IngestControlMessage, BaseModel) -> IngestControlMessage
+
+    Also allows for generic (*args, **kwargs) signatures for flexibility with class constructors.
+
+    Raises
+    ------
+    TypeError
+        If the signature does not match the expected pattern.
+    """
+    params = list(sig.parameters.values())
+
+    # If the signature accepts arbitrary keyword arguments, it's flexible enough.
+    if any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params):
+        return
+
+    if len(params) != 2:
+        raise TypeError(f"Expected exactly 2 parameters, got {len(params)}")
+
+    if params[0].name != "control_message" or params[1].name != "stage_config":
+        raise TypeError("Expected parameter names: 'control_message', 'stage_config'")
+
+    first_param = params[0].annotation
+    second_param = params[1].annotation
+    return_type = sig.return_annotation
+
+    if first_param is inspect.Parameter.empty:
+        raise TypeError("First parameter must be annotated with IngestControlMessage")
+
+    if second_param is inspect.Parameter.empty:
+        raise TypeError("Second parameter must be annotated with a subclass of BaseModel")
+
+    if return_type is inspect.Signature.empty:
+        raise TypeError("Return type must be annotated with IngestControlMessage")
+
+    if not issubclass(first_param, IngestControlMessage):
+        raise TypeError(f"First parameter must be IngestControlMessage, got {first_param}")
+
+    if not (issubclass(second_param, BaseModel)):
+        raise TypeError(f"Second parameter must be a subclass of BaseModel, got {second_param}")
+
+    if not issubclass(return_type, IngestControlMessage):
+        raise TypeError(f"Return type must be IngestControlMessage, got {return_type}")
+
+
+def ingest_callable_signature(sig: inspect.Signature):
+    """
+    Validates that a callable has the signature:
+        (IngestControlMessage) -> IngestControlMessage
+
+    Also allows for generic (*args, **kwargs) signatures for flexibility with class constructors.
+
+    Raises
+    ------
+    TypeError
+        If the signature does not match the expected pattern.
+    """
+    params = list(sig.parameters.values())
+
+    # If the signature accepts arbitrary keyword arguments, it's flexible enough.
+    if any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params):
+        return
+
+    if len(params) != 1:
+        raise TypeError(f"Expected exactly 1 parameter, got {len(params)}")
+
+    if params[0].name != "control_message":
+        raise TypeError("Expected parameter name: 'control_message'")
+
+    first_param = params[0].annotation
+    return_type = sig.return_annotation
+
+    if first_param is inspect.Parameter.empty:
+        raise TypeError("Parameter must be annotated with IngestControlMessage")
+
+    if return_type is inspect.Signature.empty:
+        raise TypeError("Return type must be annotated with IngestControlMessage")
+
+    # Handle string annotations (forward references)
+    if isinstance(first_param, str):
+        if first_param != "IngestControlMessage":
+            raise TypeError(f"Parameter must be IngestControlMessage, got {first_param}")
+    else:
+        # Handle actual class annotations
+        if not issubclass(first_param, IngestControlMessage):
+            raise TypeError(f"Parameter must be IngestControlMessage, got {first_param}")
+
+    # Handle string annotations for return type
+    if isinstance(return_type, str):
+        if return_type != "IngestControlMessage":
+            raise TypeError(f"Return type must be IngestControlMessage, got {return_type}")
+    else:
+        # Handle actual class annotations
+        if not issubclass(return_type, IngestControlMessage):
+            raise TypeError(f"Return type must be IngestControlMessage, got {return_type}")

nv_ingest_api/util/imports/dynamic_resolvers.py

@@ -0,0 +1,158 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import importlib
+import inspect
+from typing import Callable, Union, List, Optional
+
+from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
+
+
+def resolve_obj_from_path(path: str, allowed_base_paths: Optional[List[str]] = None) -> object:
+    """
+    Import and return an object from a string path of the form 'module.sub:attr'.
+
+    To enhance security, this function can restrict imports to a list of allowed base module paths.
+    """
+    if ":" not in path:
+        raise ValueError(f"Invalid path '{path}': expected format 'module.sub:attr'")
+    module_path, attr_name = path.split(":", 1)
+
+    # Security check: only allow imports from specified base paths if provided.
+    if allowed_base_paths:
+        is_allowed = any(module_path == base or module_path.startswith(base + ".") for base in allowed_base_paths)
+        if not is_allowed:
+            raise ImportError(
+                f"Module '{module_path}' is not in the list of allowed base paths. "
+                f"Allowed paths: {allowed_base_paths}"
+            )
+
+    try:
+        mod = importlib.import_module(module_path)
+    except ModuleNotFoundError as e:
+        raise ImportError(f"Could not import module '{module_path}'") from e
+    try:
+        obj = getattr(mod, attr_name)
+    except AttributeError as e:
+        raise AttributeError(f"Module '{module_path}' has no attribute '{attr_name}'") from e
+    return obj
+
+
+def resolve_callable_from_path(
+    callable_path: str,
+    signature_schema: Union[List[str], Callable[[inspect.Signature], None], str],
+    allowed_base_paths: Optional[List[str]] = None,
+) -> Callable:
+    """
+    Import and return a callable from a module path string like 'module.submodule:callable_name',
+    and validate its signature using the required signature_schema (callable or path to callable).
+
+    Parameters
+    ----------
+    callable_path : str
+        The module path and callable in the format 'module.sub:callable'.
+    signature_schema : Union[List[str], Callable, str]
+        Either:
+            - A list of parameter names to require.
+            - A callable that takes an inspect.Signature and raises on failure.
+            - A string path to such a callable ('module.sub:schema_checker').
+    allowed_base_paths : Optional[List[str]]
+        An optional list of base module paths from which imports are allowed.
+        If provided, both the callable and any signature schema specified by path
+        must reside within one of these paths.
+
+    Returns
+    -------
+    Callable
+        The resolved and validated callable.
+
+    Raises
+    ------
+    ValueError
+        If the path is not correctly formatted.
+    ImportError
+        If the module cannot be imported or is not in the allowed paths.
+    AttributeError
+        If the attribute does not exist in the module.
+    TypeError
+        If the resolved attribute is not callable or the signature does not match.
+    """
+    obj = resolve_obj_from_path(callable_path, allowed_base_paths=allowed_base_paths)
+    if not callable(obj):
+        raise TypeError(f"Object '{callable_path}' is not callable")
+
+    # Load/check signature_schema
+    schema_checker = signature_schema
+    if isinstance(signature_schema, str):
+        # When loading the schema checker, apply the same security restrictions.
+        schema_checker = resolve_obj_from_path(signature_schema, allowed_base_paths=allowed_base_paths)
+
+    sig = inspect.signature(obj)
+    if isinstance(schema_checker, list):
+        actual_params = list(sig.parameters.keys())
+        missing = [p for p in schema_checker if p not in actual_params]
+        if missing:
+            raise TypeError(
+                f"Callable at '{callable_path}' is missing required parameters: {missing}\n"
+                f"Actual parameters: {actual_params}"
+            )
+    elif callable(schema_checker):
+        try:
+            schema_checker(sig)
+        except Exception as e:
+            raise TypeError(f"Signature validation for '{callable_path}' failed: {e}") from e
+    else:
+        raise TypeError(f"Invalid signature_schema: expected list, callable, or str, got {type(signature_schema)}")
+
+    return obj
+
+
+def resolve_actor_class_from_path(
+    path: str, expected_base_class: type, allowed_base_paths: Optional[List[str]] = None
+) -> type:
+    """
+    Resolves an actor class from a path and validates that it is a class
+    that inherits from the expected base class. This function correctly handles
+    decorated Ray actors by inspecting their original class.
+
+    Parameters
+    ----------
+    path : str
+        The full import path to the actor class.
+    expected_base_class : type
+        The base class that the resolved class must inherit from.
+    allowed_base_paths : Optional[List[str]]
+        An optional list of base module paths from which imports are allowed.
+
+    Returns
+    -------
+    type
+        The resolved actor class (or Ray actor factory).
+    """
+    obj = resolve_obj_from_path(path, allowed_base_paths=allowed_base_paths)
+
+    # Determine the class to validate. If it's a Ray actor factory, we need to
+    # inspect its MRO to find the original user-defined class.
+    cls_to_validate = None
+    if inspect.isclass(obj):
+        cls_to_validate = obj
+    else:
+        # For actor factories, find the base class in the MRO that inherits from RayActorStage
+        for base in obj.__class__.__mro__:
+            if inspect.isclass(base) and issubclass(base, RayActorStage) and base is not RayActorStage:
+                cls_to_validate = base
+                break
+
+    if cls_to_validate is None:
+        raise TypeError(
+            f"Could not resolve a valid actor class from path '{path}'. "
+            f"The object is not a class and not a recognized actor factory."
+        )
+
+    if not issubclass(cls_to_validate, expected_base_class):
+        raise TypeError(
+            f"Actor class '{cls_to_validate.__name__}' at '{path}' must inherit from '{expected_base_class.__name__}'."
+        )
+
+    return obj

nv_ingest_api/util/introspection/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0

nv_ingest_api/util/introspection/class_inspect.py

@@ -0,0 +1,145 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import inspect
+from typing import Optional, Type, Union, Callable
+
+from pydantic import BaseModel
+
+
+def find_pydantic_config_schema(
+    actor_class: Type,
+    base_class_to_find: Type,
+    param_name: str = "config",
+) -> Optional[Type[BaseModel]]:
+    """
+    Introspects a class's MRO to find a Pydantic model in its __init__ signature.
+
+    This function is designed to find the specific Pydantic configuration model
+    for a pipeline actor, which might be a direct class or a proxy object.
+
+    Parameters
+    ----------
+    actor_class : Type
+        The actor class or proxy object to inspect.
+    base_class_to_find : Type
+        The specific base class (e.g., RaySource, RayStage) to look for when
+        resolving the true actor class from a proxy.
+    param_name : str, optional
+        The name of the __init__ parameter to inspect for the Pydantic schema,
+        by default "config".
+
+    Returns
+    -------
+    Optional[Type[BaseModel]]
+        The Pydantic BaseModel class if found, otherwise None.
+    """
+    # 1. Find the actual class to inspect, handling proxy objects.
+    cls_to_inspect = None
+    if inspect.isclass(actor_class):
+        cls_to_inspect = actor_class
+    else:
+        for base in actor_class.__class__.__mro__:
+            if inspect.isclass(base) and issubclass(base, base_class_to_find) and base is not base_class_to_find:
+                cls_to_inspect = base
+                break
+
+    if not cls_to_inspect:
+        return None
+
+    # 2. Walk the MRO of the real class to find the __init__ with the typed parameter.
+    for cls in cls_to_inspect.__mro__:
+        if param_name in getattr(cls.__init__, "__annotations__", {}):
+            try:
+                init_sig = inspect.signature(cls.__init__)
+                config_param = init_sig.parameters.get(param_name)
+                if (
+                    config_param
+                    and config_param.annotation is not BaseModel
+                    and issubclass(config_param.annotation, BaseModel)
+                ):
+                    return config_param.annotation  # Found the schema
+            except (ValueError, TypeError):
+                # This class's __init__ is not inspectable (e.g., a C-extension), continue up the MRO.
+                continue
+
+    return None
+
+
+def find_pydantic_config_schema_for_callable(
+    callable_fn: Callable,
+    param_name: str = "stage_config",
+) -> Optional[Type[BaseModel]]:
+    """
+    Introspects a callable's signature to find a Pydantic model parameter.
+
+    This function is designed to find the specific Pydantic configuration model
+    for a pipeline callable function.
+
+    Parameters
+    ----------
+    callable_fn : Callable
+        The callable function to inspect.
+    param_name : str, optional
+        The name of the parameter to inspect for the Pydantic schema,
+        by default "stage_config".
+
+    Returns
+    -------
+    Optional[Type[BaseModel]]
+        The Pydantic BaseModel class if found, otherwise None.
+    """
+    try:
+        sig = inspect.signature(callable_fn)
+        config_param = sig.parameters.get(param_name)
+        if (
+            config_param
+            and config_param.annotation is not BaseModel
+            and hasattr(config_param.annotation, "__mro__")
+            and issubclass(config_param.annotation, BaseModel)
+        ):
+            return config_param.annotation
+    except (ValueError, TypeError):
+        # Function signature is not inspectable
+        pass
+
+    return None
+
+
+def find_pydantic_config_schema_unified(
+    target: Union[Type, Callable],
+    base_class_to_find: Optional[Type] = None,
+    param_name: str = "config",
+) -> Optional[Type[BaseModel]]:
+    """
+    Unified function to find Pydantic schema for either classes or callables.
+
+    Parameters
+    ----------
+    target : Union[Type, Callable]
+        The class or callable to inspect.
+    base_class_to_find : Optional[Type], optional
+        The specific base class to look for when resolving actor classes from proxies.
+        Only used for class inspection.
+    param_name : str, optional
+        The name of the parameter to inspect for the Pydantic schema.
+        For classes: defaults to "config"
+        For callables: should be "stage_config"
+
+    Returns
+    -------
+    Optional[Type[BaseModel]]
+        The Pydantic BaseModel class if found, otherwise None.
+    """
+    if callable(target) and not inspect.isclass(target):
+        # Handle callable function
+        return find_pydantic_config_schema_for_callable(target, param_name)
+    elif inspect.isclass(target) or hasattr(target, "__class__"):
+        # Handle class or proxy object
+        if base_class_to_find is None:
+            # If no base class specified, we can't use the original function
+            return None
+        return find_pydantic_config_schema(target, base_class_to_find, param_name)
+    else:
+        return None

nv_ingest_api/util/introspection/function_inspect.py

@@ -0,0 +1,65 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Utilities for introspecting and analyzing UDF function specifications.
+"""
+
+import re
+from typing import Optional
+
+
+def infer_udf_function_name(udf_function: str) -> Optional[str]:
+    """
+    Attempts to infer the UDF function name from the provided function string.
+
+    Supports three formats:
+    1. Inline function: 'def my_func(control_message): ...' -> 'my_func'
+    2. Import path: 'my_module.my_function' -> 'my_function'
+    3. File path: '/path/to/file.py:function_name' -> 'function_name'
+
+    Parameters
+    ----------
+    udf_function : str
+        The UDF function string.
+
+    Returns
+    -------
+    Optional[str]
+        The inferred UDF function name, or None if inference is not possible.
+
+    Examples
+    --------
+    >>> infer_udf_function_name("def my_custom_func(control_message): pass")
+    'my_custom_func'
+
+    >>> infer_udf_function_name("my_module.submodule.process_data")
+    'process_data'
+
+    >>> infer_udf_function_name("/path/to/script.py:custom_function")
+    'custom_function'
+
+    >>> infer_udf_function_name("/path/to/script.py")
+    None
+    """
+    udf_function = udf_function.strip()
+
+    # Format 3: File path with explicit function name
+    if ":" in udf_function and ("/" in udf_function or "\\" in udf_function):
+        # File path with explicit function name: '/path/to/file.py:function_name'
+        return udf_function.split(":")[-1].strip()
+
+    # Format 2: Import path like 'module.submodule.function'
+    elif "." in udf_function and not udf_function.startswith("def "):
+        # Import path: extract the last part as function name
+        return udf_function.split(".")[-1].strip()
+
+    # Format 1: Inline function definition
+    elif udf_function.startswith("def "):
+        # Parse inline function definition to extract function name
+        match = re.match(r"def\s+(\w+)\s*\(", udf_function)
+        if match:
+            return match.group(1)
+
+    return None

nv_ingest_api/util/logging/configuration.py

@@ -0,0 +1,102 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+
+import logging
+import logging.config
+from enum import Enum
+
+
+class LogLevel(str, Enum):
+    DEFAULT = "DEFAULT"
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    CRITICAL = "CRITICAL"
+
+
+def configure_logging(level_name: str) -> None:
+    """
+    Configures global logging.
+
+    Parameters
+    ----------
+    level_name : str
+        The name of the logging level (e.g., "DEBUG", "INFO").
+    """
+    numeric_level = getattr(logging, level_name.upper(), None)
+    if not isinstance(numeric_level, int):
+        raise ValueError(f"Invalid log level: {level_name}")
+
+    # Scorched-earth reset: remove ALL existing handlers from root and named loggers
+    # to ensure there is exactly one handler after configuration.
+    root_logger = logging.getLogger()
+    for h in list(root_logger.handlers):
+        root_logger.removeHandler(h)
+        try:
+            h.close()
+        except Exception:
+            pass
+
+    # Clear handlers from all known loggers and make them propagate to root
+    for name, logger_obj in list(logging.Logger.manager.loggerDict.items()):
+        if isinstance(logger_obj, logging.Logger):
+            for h in list(logger_obj.handlers):
+                logger_obj.removeHandler(h)
+                try:
+                    h.close()
+                except Exception:
+                    pass
+            # Ensure messages bubble to root; levels will be controlled centrally
+            logger_obj.propagate = True
+            logger_obj.setLevel(logging.NOTSET)
+
+    # Use dictConfig to establish a single console handler on the root logger.
+    config_dict = {
+        "version": 1,
+        # We already cleared handlers above; keep loggers enabled so they propagate to root
+        "disable_existing_loggers": False,
+        "formatters": {
+            "standard": {
+                "format": "%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+            }
+        },
+        "handlers": {
+            "console": {
+                "class": "logging.StreamHandler",
+                "level": numeric_level,
+                "formatter": "standard",
+                "stream": "ext://sys.stdout",
+            }
+        },
+        "root": {
+            "level": numeric_level,
+            "handlers": ["console"],
+        },
+    }
+
+    logging.config.dictConfig(config_dict)
+
+    # Enforce exactly one handler remains attached to root (keep first StreamHandler)
+    root_logger = logging.getLogger()
+    if len(root_logger.handlers) > 1:
+        keep = None
+        for h in list(root_logger.handlers):
+            if keep is None and isinstance(h, logging.StreamHandler):
+                keep = h
+                continue
+            root_logger.removeHandler(h)
+            try:
+                h.close()
+            except Exception:
+                pass
+
+    # Route warnings module through logging
+    try:
+        import logging as _logging
+
+        _logging.captureWarnings(True)
+    except Exception:
+        pass

nv_ingest_api/util/logging/sanitize.py

@@ -0,0 +1,84 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from typing import Any, Mapping, MutableMapping, Sequence, Set
+
+try:
+    # Pydantic is optional at runtime for this helper; import if available
+    from pydantic import BaseModel  # type: ignore
+except Exception:  # pragma: no cover - pydantic always present in this repo
+    BaseModel = None  # type: ignore
+
+
+_DEFAULT_SENSITIVE_KEYS: Set[str] = {
+    "access_token",
+    "api_key",
+    "authorization",
+    "auth_token",
+    "client_secret",
+    "hf_access_token",
+    "hugging_face_access_token",
+    "password",
+    "refresh_token",
+    "secret",
+    "ssl_cert",
+    "x-api-key",
+}
+
+_REDACTION = "***REDACTED***"
+
+
+def _is_mapping(obj: Any) -> bool:
+    try:
+        return isinstance(obj, Mapping)
+    except Exception:
+        return False
+
+
+def _is_sequence(obj: Any) -> bool:
+    # Exclude strings/bytes from sequences we want to traverse
+    return isinstance(obj, Sequence) and not isinstance(obj, (str, bytes, bytearray))
+
+
+def sanitize_for_logging(
+    data: Any,
+    sensitive_keys: Set[str] | None = None,
+    redaction: str = _REDACTION,
+) -> Any:
+    """
+    Recursively sanitize common secret fields from dicts, lists, tuples, and Pydantic models.
+
+    - Key comparison is case-insensitive and matches exact keys only.
+    - Does not mutate input; returns a sanitized deep copy.
+    - For Pydantic BaseModel instances, uses model_dump() before redaction.
+    """
+    keys = {k.lower() for k in (sensitive_keys or _DEFAULT_SENSITIVE_KEYS)}
+
+    # Handle Pydantic models without importing pydantic at module import time
+    if BaseModel is not None and isinstance(data, BaseModel):  # type: ignore[arg-type]
+        try:
+            return sanitize_for_logging(data.model_dump(), keys, redaction)
+        except Exception:
+            # Fall through and try generic handling below
+            pass
+
+    # Dict-like
+    if _is_mapping(data):
+        out: MutableMapping[str, Any] = type(data)()  # preserve mapping type where possible
+        for k, v in data.items():  # type: ignore[assignment]
+            key_lower = str(k).lower()
+            if key_lower in keys:
+                out[k] = redaction
+            else:
+                out[k] = sanitize_for_logging(v, keys, redaction)
+        return out
+
+    # List/Tuple/Sequence
+    if _is_sequence(data):
+        return type(data)(sanitize_for_logging(v, keys, redaction) for v in data)
+
+    # Fallback: return as-is
+    return data

nv_ingest_api/util/message_brokers/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0