nv-ingest-api 2025.4.21.dev20250421__py3-none-any.whl → 2025.4.22.dev20250422__py3-none-any.whl

nv_ingest_api/util/detectors/language.py

@@ -0,0 +1,38 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+
+import langdetect
+
+from nv_ingest_api.internal.enums.common import LanguageEnum
+from nv_ingest_api.util.exception_handlers.detectors import langdetect_exception_handler
+
+
+@langdetect_exception_handler
+def detect_language(text):
+    """
+    Detect spoken language from a string of text.
+
+    Parameters
+    ----------
+    text : str
+        A string of text.
+
+    Returns
+    -------
+    LanguageEnum
+        A value from `LanguageEnum` detected language code.
+    """
+
+    try:
+        language = langdetect.detect(text)
+
+        if LanguageEnum.has_value(language):
+            language = LanguageEnum[language.upper().replace("-", "_")]
+        else:
+            language = LanguageEnum.UNKNOWN
+    except langdetect.lang_detect_exception.LangDetectException:
+        language = LanguageEnum.UNKNOWN
+
+    return language

nv_ingest_api/util/exception_handlers/converters.py

@@ -0,0 +1,72 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+
+import logging
+from datetime import datetime
+from datetime import timezone
+from typing import Any
+from typing import Callable
+from typing import Dict
+
+from nv_ingest_api.util.converters import datetools
+
+logger = logging.getLogger(__name__)
+
+
+def datetools_exception_handler(func: Callable, **kwargs: Dict[str, Any]) -> Callable:
+    """
+    A decorator that handles exceptions for date-related functions.
+
+    This decorator wraps a function that processes dates and catches any exceptions that occur during its execution.
+    If an exception is raised, it logs a warning and returns the current UTC time as an ISO 8601 formatted string.
+
+    Parameters
+    ----------
+    func : Callable
+        The function to be decorated. This function is expected to handle date operations.
+
+    kwargs : dict
+        Additional keyword arguments to be passed to the function.
+
+    Returns
+    -------
+    Callable
+        The wrapped function that executes `func` with exception handling.
+
+    Notes
+    -----
+    If an exception is raised while executing the wrapped function, the current UTC time (with timezone information
+    removed)
+    will be returned as an ISO 8601 formatted string.
+
+    Examples
+    --------
+    >>> @datetools_exception_handler
+    ... def parse_date(date_str):
+    ...     return datetime.strptime(date_str, '%Y-%m-%d')
+    ...
+    >>> parse_date('2024-08-22')
+    datetime.datetime(2024, 8, 22, 0, 0)
+
+    If the input is invalid, the current UTC time without timezone information is returned:
+
+    >>> parse_date('invalid-date')
+    '2024-08-22T12:34:56'
+
+    Raises
+    ------
+    Exception
+        Any exception raised by the wrapped function is caught, logged, and handled by returning the current UTC time.
+    """
+
+    def inner_function(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            log_error_message = f"Invalid date format: {e}"
+            logger.warning(log_error_message)
+            return datetools.remove_tz(datetime.now(timezone.utc)).isoformat()
+
+    return inner_function

nv_ingest_api/util/exception_handlers/decorators.py

@@ -0,0 +1,223 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import functools
+import inspect
+import re
+import typing
+from functools import wraps
+
+from nv_ingest_api.internal.primitives.ingest_control_message import IngestControlMessage
+from nv_ingest_api.internal.primitives.tracing.logging import TaskResultStatus, annotate_task_result
+from nv_ingest_api.util.control_message.validators import cm_ensure_payload_not_null, cm_set_failure
+
+
+logger = logging.getLogger(__name__)
+
+
+# TODO(Devin): move back to framework
+def nv_ingest_node_failure_context_manager(
+    annotation_id: str,
+    payload_can_be_empty: bool = False,
+    raise_on_failure: bool = False,
+    skip_processing_if_failed: bool = True,
+    forward_func=None,
+) -> typing.Callable:
+    """
+    A decorator that applies a default failure context manager around a function to manage
+    the execution and potential failure of operations involving IngestControlMessages.
+
+    Parameters
+    ----------
+    annotation_id : str
+        A unique identifier used for annotating the task's result.
+    payload_can_be_empty : bool, optional
+        If False, the payload of the IngestControlMessage will be checked to ensure it's not null,
+        raising an exception if it is null. Defaults to False, enforcing payload presence.
+    raise_on_failure : bool, optional
+        If True, an exception is raised if the decorated function encounters an error.
+        Otherwise, the error is handled silently by annotating the IngestControlMessage. Defaults to False.
+    skip_processing_if_failed : bool, optional
+        If True, skips the processing of the decorated function if the control message has already
+        been marked as failed. If False, the function will be processed regardless of the failure
+        status of the IngestControlMessage. Defaults to True.
+    forward_func : callable, optional
+        A function to forward the IngestControlMessage if it has already been marked as failed.
+
+    Returns
+    -------
+    Callable
+        A decorator that wraps the given function with failure handling logic.
+    """
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(control_message: IngestControlMessage, *args, **kwargs):
+            # Quick return if the IngestControlMessage has already failed
+            is_failed = control_message.get_metadata("cm_failed", False)
+            if not is_failed or not skip_processing_if_failed:
+                with CMNVIngestFailureContextManager(
+                    control_message=control_message,
+                    annotation_id=annotation_id,
+                    raise_on_failure=raise_on_failure,
+                    func_name=func.__name__,
+                ) as ctx_mgr:
+                    if not payload_can_be_empty:
+                        cm_ensure_payload_not_null(control_message=control_message)
+                    control_message = func(ctx_mgr.control_message, *args, **kwargs)
+            else:
+                if forward_func:
+                    control_message = forward_func(control_message)
+            return control_message
+
+        return wrapper
+
+    return decorator
+
+
+def nv_ingest_source_failure_context_manager(
+    annotation_id: str,
+    payload_can_be_empty: bool = False,
+    raise_on_failure: bool = False,
+) -> typing.Callable:
+    """
+    A decorator that ensures any function's output is treated as a IngestControlMessage for annotation.
+    It applies a context manager to handle success and failure annotations based on the function's execution.
+
+    Parameters
+    ----------
+    annotation_id : str
+        Unique identifier used for annotating the function's output.
+    payload_can_be_empty : bool, optional
+        Specifies if the function's output IngestControlMessage payload can be empty, default is False.
+    raise_on_failure : bool, optional
+        Determines if an exception should be raised upon function failure, default is False.
+
+    Returns
+    -------
+    Callable
+        A decorator that ensures function output is processed for success or failure annotation.
+    """
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs) -> IngestControlMessage:
+            try:
+                result = func(*args, **kwargs)
+                if not isinstance(result, IngestControlMessage):
+                    raise TypeError(f"{func.__name__} output is not a IngestControlMessage as expected.")
+                if not payload_can_be_empty and result.get_metadata("payload") is None:
+                    raise ValueError(f"{func.__name__} IngestControlMessage payload cannot be null.")
+
+                # Success annotation.
+                annotate_task_result(result, result=TaskResultStatus.SUCCESS, task_id=annotation_id)
+            except Exception as e:
+                error_message = f"Error in {func.__name__}: {e}"
+                # Prepare a new IngestControlMessage for failure annotation if needed.
+                if "result" not in locals() or not isinstance(result, IngestControlMessage):
+                    result = IngestControlMessage()
+                cm_set_failure(result, error_message)
+                annotate_task_result(
+                    result,
+                    result=TaskResultStatus.FAILURE,
+                    task_id=annotation_id,
+                    message=error_message,
+                )
+                if raise_on_failure:
+                    raise
+            return result
+
+        return wrapper
+
+    return decorator
+
+
+class CMNVIngestFailureContextManager:
+    """
+    Context manager for handling IngestControlMessage failures during processing, providing
+    a structured way to annotate and manage failures and successes.
+
+    Parameters
+    ----------
+    control_message : IngestControlMessage
+        The IngestControlMessage instance to be managed.
+    annotation_id : str
+        The task's unique identifier for annotation purposes.
+    raise_on_failure : bool, optional
+        Determines whether to raise an exception upon failure. Defaults to False, which
+        means failures are annotated rather than raising exceptions.
+    func_name : str, optional
+        The name of the function being wrapped, used to annotate error messages uniformly.
+        If None, stack introspection is used to deduce a likely function name. Defaults to None.
+
+    Returns
+    -------
+    None
+    """
+
+    def __init__(
+        self,
+        control_message: IngestControlMessage,
+        annotation_id: str,
+        raise_on_failure: bool = False,
+        func_name: str = None,
+    ):
+        self.control_message = control_message
+        self.annotation_id = annotation_id
+        self.raise_on_failure = raise_on_failure
+        if func_name is not None:
+            self._func_name = func_name
+        else:
+            try:
+                # Use stack introspection to get a candidate function name.
+                stack = inspect.stack()
+                # Use the third frame as a heuristic; adjust if needed.
+                candidate = stack[2].function if len(stack) > 2 else "UnknownFunction"
+                # Remove any whitespace and limit the length to 50 characters.
+                candidate = re.sub(r"\s+", "", candidate)[:50]
+                self._func_name = candidate if candidate else "UnknownFunction"
+            except Exception:
+                self._func_name = "UnknownFunction"
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if exc_type is not None:  # An exception occurred
+            error_message = f"Error in {self._func_name}: {exc_value}"
+            if self.control_message is not None:
+                cm_set_failure(self.control_message, error_message)
+                annotate_task_result(
+                    self.control_message,
+                    result=TaskResultStatus.FAILURE,
+                    task_id=self.annotation_id,
+                    message=error_message,
+                )
+            # Propagate the exception if raise_on_failure is True; otherwise, suppress it.
+            if self.raise_on_failure:
+                return False
+            return True
+
+        annotate_task_result(
+            self.control_message,
+            result=TaskResultStatus.SUCCESS,
+            task_id=self.annotation_id,
+        )
+        return False
+
+
+def unified_exception_handler(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            # Use the function's name in the error message
+            func_name = func.__name__
+            err_msg = f"{func_name}: error: {e}"
+            logger.exception(err_msg, exc_info=True)
+            raise type(e)(err_msg) from e
+
+    return wrapper

nv_ingest_api/util/exception_handlers/detectors.py

@@ -0,0 +1,74 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+
+import logging
+from typing import Any
+from typing import Callable
+from typing import Dict
+
+from langdetect.lang_detect_exception import LangDetectException
+
+from nv_ingest_api.internal.enums.common import LanguageEnum
+
+logger = logging.getLogger(__name__)
+
+
+def langdetect_exception_handler(func: Callable, **kwargs: Dict[str, Any]) -> Callable:
+    """
+    A decorator that handles `LangDetectException` for language detection functions.
+
+    This decorator wraps a function that performs language detection and catches any `LangDetectException` that occurs
+    during its execution.
+    If such an exception is raised, it logs a warning and returns a default value of `LanguageEnum.UNKNOWN`.
+
+    Parameters
+    ----------
+    func : callable
+        The function to be decorated. This function is expected to handle language detection.
+
+    kwargs : dict
+        Additional keyword arguments to be passed to the function.
+
+    Returns
+    -------
+    callable
+        The wrapped function that executes `func` with exception handling.
+
+    Notes
+    -----
+    If a `LangDetectException` is raised while executing the wrapped function, the exception is logged,
+    and `LanguageEnum.UNKNOWN` is returned as a fallback value.
+
+    Examples
+    --------
+    >>> @langdetect_exception_handler
+    ... def detect_language(text):
+    ...     # Function implementation here
+    ...     pass
+    ...
+    >>> detect_language('This is a test sentence.')
+    <LanguageEnum.EN: 'en'>
+
+    If a `LangDetectException` is encountered, the function will return `LanguageEnum.UNKNOWN`:
+
+    >>> detect_language('')
+    <LanguageEnum.UNKNOWN: 'unknown'>
+
+    Raises
+    ------
+    LangDetectException
+        The exception raised by the wrapped function is caught and handled by logging a warning
+        and returning `LanguageEnum.UNKNOWN`.
+    """
+
+    def inner_function(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except LangDetectException as e:
+            log_error_message = f"LangDetectException: {e}"
+            logger.warning(log_error_message)
+            return LanguageEnum.UNKNOWN
+
+    return inner_function

nv_ingest_api/util/exception_handlers/pdf.py

@@ -0,0 +1,116 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+
+import logging
+
+from nv_ingest_api.internal.enums.common import StatusEnum, TaskTypeEnum
+from nv_ingest_api.internal.schemas.meta.metadata_schema import validate_metadata
+
+logger = logging.getLogger(__name__)
+
+
+def pdfium_exception_handler(descriptor):
+    """
+    A decorator that handles exceptions for functions interacting with PDFium.
+
+    This decorator wraps a function and catches any exceptions that occur during its execution.
+    If an exception is raised, it logs a warning with a descriptor and the function name,
+    then returns an empty list as a fallback value.
+
+    Parameters
+    ----------
+    descriptor : str
+        A string descriptor to identify the context or source of the function being wrapped.
+        This descriptor is included in the log message if an exception occurs.
+
+    Returns
+    -------
+    callable
+        A decorator function that wraps the target function with exception handling.
+
+    Notes
+    -----
+    This decorator is useful for ensuring that functions interacting with PDFium can gracefully handle errors
+    without interrupting the entire processing pipeline.
+
+    Examples
+    --------
+    >>> @pdfium_exception_handler("PDF Processing")
+    ... def process_pdf(file_path):
+    ...     # Function implementation here
+    ...     pass
+    ...
+    >>> process_pdf("example.pdf")
+    []
+
+    Raises
+    ------
+    Exception
+        Any exception raised by the wrapped function is caught, logged, and handled by returning an empty list.
+    """
+
+    def outer_function(func):
+        def inner_function(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except Exception as e:
+                log_error_message = f"{descriptor}:{func.__name__} error:{e}"
+                logger.warning(log_error_message)
+                return []
+
+        return inner_function
+
+    return outer_function
+
+
+def create_exception_tag(error_message, source_id=None):
+    """
+    Creates a metadata tag for logging or reporting an exception.
+
+    This function generates a metadata dictionary containing information about the exception,
+    including the task type, status, source identifier, and error message.
+    The metadata is validated and returned as a list containing a single entry.
+
+    Parameters
+    ----------
+    error_message : str
+        The error message describing the exception.
+    source_id : Optional[str], default=None
+        The identifier for the source related to the error, if available.
+
+    Returns
+    -------
+    list
+        A list containing a single entry, which is a tuple. The first element of the tuple is `None`,
+        and the second element is the validated metadata dictionary as a `dict`.
+
+    Notes
+    -----
+    This function is typically used to generate error metadata for tracking and logging purposes.
+
+    Examples
+    --------
+    >>> create_exception_tag("File not found", source_id="12345")
+    [[None, {'task': 'EXTRACT', 'status': 'ERROR', 'source_id': '12345', 'error_msg': 'File not found'}]]
+
+    Raises
+    ------
+    ValidationError
+        If the metadata does not pass validation.
+    """
+    unified_metadata = {}
+
+    error_metadata = {
+        "task": TaskTypeEnum.EXTRACT,
+        "status": StatusEnum.ERROR,
+        "source_id": source_id,
+        "error_msg": error_message,
+    }
+
+    unified_metadata["error_metadata"] = error_metadata
+
+    validated_unified_metadata = validate_metadata(unified_metadata)
+
+    return [[None, validated_unified_metadata.model_dump()]]

nv_ingest_api/util/exception_handlers/schemas.py

@@ -0,0 +1,68 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+
+import logging
+
+from pydantic import ValidationError
+
+logger = logging.getLogger(__name__)
+
+
+def schema_exception_handler(func, **kwargs):
+    """
+    A decorator that handles `ValidationError` exceptions for schema validation functions.
+
+    This decorator wraps a function that performs schema validation using Pydantic.
+    If a `ValidationError` is raised, it logs detailed error messages and raises a `ValueError` with the combined error
+    messages.
+
+    Parameters
+    ----------
+    func : callable
+        The function to be decorated. This function is expected to perform schema validation.
+
+    kwargs : dict
+        Additional keyword arguments to be passed to the function.
+
+    Returns
+    -------
+    callable
+        The wrapped function that executes `func` with exception handling.
+
+    Raises
+    ------
+    ValueError
+        If a `ValidationError` is caught, this decorator logs the error details and raises a `ValueError` with the
+        combined error messages.
+
+    Notes
+    -----
+    This decorator is particularly useful for functions that validate configurations or data models,
+    ensuring that any validation errors are logged and communicated clearly.
+
+    Examples
+    --------
+    >>> @schema_exception_handler
+    ... def validate_config(config_data):
+    ...     schema = MySchema(**config_data)
+    ...     return schema
+    ...
+    >>> try:
+    ...     validate_config(invalid_config)
+    ... except ValueError as e:
+    ...     print(f"Caught error: {e}")
+    Caught error: Invalid configuration: field1: value is not a valid integer; field2: field required
+    """
+
+    def inner_function(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except ValidationError as e:
+            error_messages = "; ".join([f"{error['loc'][0]}: {error['msg']}" for error in e.errors()])
+            log_error_message = f"Invalid configuration: {error_messages}"
+            logger.error(log_error_message)
+            raise ValueError(log_error_message)
+
+    return inner_function

nv_ingest_api/util/image_processing/__init__.py

@@ -0,0 +1,5 @@
+from .transforms import scale_image_to_encoding_size
+
+__all__ = [
+    "scale_image_to_encoding_size",
+]