nv-ingest-api 2025.4.18.dev20250418__py3-none-any.whl → 2025.4.20.dev20250420__py3-none-any.whl

nv_ingest_api/internal/primitives/tracing/tagging.py

@@ -1,197 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
-# All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-
-import functools
-import inspect
-import string
-from datetime import datetime
-
-
-def traceable(trace_name=None):
-    """
-    A decorator that adds entry and exit trace timestamps to a IngestControlMessage's metadata
-    based on the presence of a 'config::add_trace_tagging' flag.
-
-    This decorator checks if the 'config::add_trace_tagging' flag is set to True in the
-    message's metadata. If so, it records the entry and exit timestamps of the function
-    execution, using either a provided custom trace name or the function's name by default.
-
-    Parameters
-    ----------
-    trace_name : str, optional
-        A custom name for the trace entries in the message metadata. If not provided, the
-        function's name is used by default.
-
-    Returns
-    -------
-    decorator_trace_tagging : Callable
-        A wrapper function that decorates the target function to implement trace tagging.
-
-    Notes
-    -----
-    The decorated function must accept a IngestControlMessage object as its first argument. The
-    IngestControlMessage object must implement `has_metadata`, `get_metadata`, and `set_metadata`
-    methods used by the decorator to check for the trace tagging flag and to add trace metadata.
-
-    The trace metadata added by the decorator includes two entries:
-    - 'trace::entry::<trace_name>': The monotonic timestamp marking the function's entry.
-    - 'trace::exit::<trace_name>': The monotonic timestamp marking the function's exit.
-
-    Example
-    -------
-    Applying the decorator without a custom trace name:
-
-    >>> @traceable()
-    ... def process_message(message):
-    ...     pass
-
-    Applying the decorator with a custom trace name:
-
-    >>> @traceable(custom_trace_name="CustomTraceName")
-    ... def process_message(message):
-    ...     pass
-
-    In both examples, `process_message` will have entry and exit timestamps added to the
-    IngestControlMessage's metadata if 'config::add_trace_tagging' is True.
-
-    """
-
-    def decorator_trace_tagging(func):
-        @functools.wraps(func)
-        def wrapper_trace_tagging(*args, **kwargs):
-            # Assuming the first argument is always the message
-            ts_fetched = datetime.now()
-            message = args[0]
-
-            do_trace_tagging = (message.has_metadata("config::add_trace_tagging") is True) and (
-                message.get_metadata("config::add_trace_tagging") is True
-            )
-
-            trace_prefix = trace_name if trace_name else func.__name__
-
-            if do_trace_tagging:
-                ts_send = message.get_timestamp("latency::ts_send")
-                ts_entry = datetime.now()
-                message.set_timestamp(f"trace::entry::{trace_prefix}", ts_entry)
-                if ts_send:
-                    message.set_timestamp(f"trace::entry::{trace_prefix}_channel_in", ts_send)
-                    message.set_timestamp(f"trace::exit::{trace_prefix}_channel_in", ts_fetched)
-
-            # Call the decorated function
-            result = func(*args, **kwargs)
-
-            if do_trace_tagging:
-                ts_exit = datetime.now()
-                message.set_timestamp(f"trace::exit::{trace_prefix}", ts_exit)
-                message.set_timestamp("latency::ts_send", ts_exit)
-
-            return result
-
-        return wrapper_trace_tagging
-
-    return decorator_trace_tagging
-
-
-def traceable_func(trace_name=None, dedupe=True):
-    """
-    A decorator that injects trace information for tracking the execution of a function.
-    It logs the entry and exit timestamps of the function in a `trace_info` dictionary,
-    which can be used for performance monitoring or debugging purposes.
-
-    Parameters
-    ----------
-    trace_name : str, optional
-        An optional string used as the prefix for the trace log entries. If not provided,
-        the decorated function's name is used. The string can include placeholders (e.g.,
-        "pdf_extractor::{model_name}") that will be dynamically replaced with matching
-        function argument values.
-    dedupe : bool, optional
-        If True, ensures that the trace entry and exit keys are unique by appending an index
-        (e.g., `_0`, `_1`) to the keys if duplicate entries are detected. Default is True.
-
-    Returns
-    -------
-    function
-        A wrapped function that injects trace information before and after the function's
-        execution.
-
-    Notes
-    -----
-    - If `trace_info` is not provided in the keyword arguments, a new dictionary is created
-      and used for storing trace entries.
-    - If `trace_name` contains format placeholders, the decorator attempts to populate them
-      with matching argument values from the decorated function.
-    - The trace information is logged in the format:
-        - `trace::entry::{trace_name}` for the entry timestamp.
-        - `trace::exit::{trace_name}` for the exit timestamp.
-    - If `dedupe` is True, the trace keys will be appended with an index to avoid
-      overwriting existing entries.
-
-    Example
-    -------
-    >>> @traceable_func(trace_name="pdf_extractor::{model_name}")
-    >>> def extract_pdf(model_name):
-    ...     pass
-    >>> trace_info = {}
-    >>> extract_pdf("my_model", trace_info=trace_info)
-
-    In this example, `model_name` is dynamically replaced in the trace_name, and the
-    trace information is logged with unique keys if deduplication is enabled.
-    """
-
-    def decorator_inject_trace_info(func):
-        @functools.wraps(func)
-        def wrapper_inject_trace_info(*args, **kwargs):
-            trace_info = kwargs.pop("trace_info", None)
-            if trace_info is None:
-                trace_info = {}
-            trace_prefix = trace_name if trace_name else func.__name__
-
-            arg_names = list(inspect.signature(func).parameters)
-            args_name_to_val = dict(zip(arg_names, args))
-
-            # If `trace_name` is a formattable string, e.g., "pdf_extractor::{model_name}",
-            # search `args` and `kwargs` to replace the placeholder.
-            placeholders = [x[1] for x in string.Formatter().parse(trace_name) if x[1] is not None]
-            if placeholders:
-                format_kwargs = {}
-                for name in placeholders:
-                    if name in args_name_to_val:
-                        arg_val = args_name_to_val[name]
-                    elif name in kwargs:
-                        arg_val = kwargs.get(name)
-                    else:
-                        arg_val = name
-                    format_kwargs[name] = arg_val
-                trace_prefix = trace_prefix.format(**format_kwargs)
-
-            trace_entry_key = f"trace::entry::{trace_prefix}"
-            trace_exit_key = f"trace::exit::{trace_prefix}"
-
-            ts_entry = datetime.now()
-
-            if dedupe:
-                trace_entry_key += "_{}"
-                trace_exit_key += "_{}"
-                i = 0
-                while (trace_entry_key.format(i) in trace_info) or (trace_exit_key.format(i) in trace_info):
-                    i += 1
-                trace_entry_key = trace_entry_key.format(i)
-                trace_exit_key = trace_exit_key.format(i)
-
-            trace_info[trace_entry_key] = ts_entry
-
-            # Call the decorated function
-            result = func(*args, **kwargs)
-
-            ts_exit = datetime.now()
-
-            trace_info[trace_exit_key] = ts_exit
-
-            return result
-
-        return wrapper_inject_trace_info
-
-    return decorator_inject_trace_info

nv_ingest_api/internal/schemas/__init__.py

@@ -1,3 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
-# All rights reserved.
-# SPDX-License-Identifier: Apache-2.0

nv_ingest_api/internal/schemas/extract/__init__.py

@@ -1,3 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
-# All rights reserved.
-# SPDX-License-Identifier: Apache-2.0

nv_ingest_api/internal/schemas/extract/extract_audio_schema.py

@@ -1,130 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
-# All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-
-import logging
-from typing import Optional
-from typing import Tuple
-
-from pydantic import BaseModel
-from pydantic import root_validator
-
-logger = logging.getLogger(__name__)
-
-
-class AudioConfigSchema(BaseModel):
-    """
-    Configuration schema for audio extraction endpoints and options.
-
-    Parameters
-    ----------
-    auth_token : Optional[str], default=None
-        Authentication token required for secure services.
-
-    audio_endpoints : Tuple[str, str]
-        A tuple containing the gRPC and HTTP services for the audio_retriever endpoint.
-        Either the gRPC or HTTP service can be empty, but not both.
-
-    Methods
-    -------
-    validate_endpoints(values)
-        Validates that at least one of the gRPC or HTTP services is provided for each endpoint.
-
-    Raises
-    ------
-    ValueError
-        If both gRPC and HTTP services are empty for any endpoint.
-
-    Config
-    ------
-    extra : str
-        Pydantic config option to forbid extra fields.
-    """
-
-    auth_token: Optional[str] = None
-    audio_endpoints: Tuple[Optional[str], Optional[str]] = (None, None)
-    audio_infer_protocol: Optional[str] = None
-    function_id: Optional[str] = None
-    use_ssl: Optional[bool] = None
-    ssl_cert: Optional[str] = None
-
-    @root_validator(pre=True)
-    def validate_endpoints(cls, values):
-        """
-        Validates the gRPC and HTTP services for all endpoints.
-
-        Parameters
-        ----------
-        values : dict
-            Dictionary containing the values of the attributes for the class.
-
-        Returns
-        -------
-        dict
-            The validated dictionary of values.
-
-        Raises
-        ------
-        ValueError
-            If both gRPC and HTTP services are empty for any endpoint.
-        """
-
-        def clean_service(service):
-            """Set service to None if it's an empty string or contains only spaces or quotes."""
-            if service is None or not service.strip() or service.strip(" \"'") == "":
-                return None
-            return service
-
-        endpoint_name = "audio_endpoints"
-        grpc_service, http_service = values.get(endpoint_name)
-        grpc_service = clean_service(grpc_service)
-        http_service = clean_service(http_service)
-
-        if not grpc_service and not http_service:
-            raise ValueError(f"Both gRPC and HTTP services cannot be empty for {endpoint_name}.")
-
-        values[endpoint_name] = (grpc_service, http_service)
-
-        protocol_name = "audio_infer_protocol"
-        protocol_value = values.get(protocol_name)
-
-        if not protocol_value:
-            protocol_value = "http" if http_service else "grpc" if grpc_service else ""
-
-        protocol_value = protocol_value.lower()
-        values[protocol_name] = protocol_value
-
-        return values
-
-    class Config:
-        extra = "forbid"
-
-
-class AudioExtractorSchema(BaseModel):
-    """
-    Configuration schema for the PDF extractor settings.
-
-    Parameters
-    ----------
-    max_queue_size : int, default=1
-        The maximum number of items allowed in the processing queue.
-
-    n_workers : int, default=16
-        The number of worker threads to use for processing.
-
-    raise_on_failure : bool, default=False
-        A flag indicating whether to raise an exception on processing failure.
-
-    audio_extraction_config: Optional[AudioConfigSchema], default=None
-        Configuration schema for the audio extraction stage.
-    """
-
-    max_queue_size: int = 1
-    n_workers: int = 16
-    raise_on_failure: bool = False
-
-    audio_extraction_config: Optional[AudioConfigSchema] = None
-
-    class Config:
-        extra = "forbid"

nv_ingest_api/internal/schemas/extract/extract_chart_schema.py

@@ -1,135 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
-# All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-import logging
-from typing import Optional
-from typing import Tuple
-
-from pydantic import field_validator, model_validator, ConfigDict, BaseModel
-
-logger = logging.getLogger(__name__)
-
-
-class ChartExtractorConfigSchema(BaseModel):
-    """
-    Configuration schema for chart extraction service endpoints and options.
-
-    Parameters
-    ----------
-    auth_token : Optional[str], default=None
-        Authentication token required for secure services.
-
-    yolox_endpoints : Tuple[Optional[str], Optional[str]], default=(None, None)
-        A tuple containing the gRPC and HTTP services for the yolox endpoint.
-        Either the gRPC or HTTP service can be empty, but not both.
-
-    paddle_endpoints : Tuple[Optional[str], Optional[str]], default=(None, None)
-        A tuple containing the gRPC and HTTP services for the paddle endpoint.
-        Either the gRPC or HTTP service can be empty, but not both.
-
-    Methods
-    -------
-    validate_endpoints(values)
-        Validates that at least one of the gRPC or HTTP services is provided for each endpoint.
-
-    Raises
-    ------
-    ValueError
-        If both gRPC and HTTP services are empty for any endpoint.
-
-    Config
-    ------
-    extra : str
-        Pydantic config option to forbid extra fields.
-    """
-
-    auth_token: Optional[str] = None
-
-    yolox_endpoints: Tuple[Optional[str], Optional[str]] = (None, None)
-    yolox_infer_protocol: str = ""
-
-    paddle_endpoints: Tuple[Optional[str], Optional[str]] = (None, None)
-    paddle_infer_protocol: str = ""
-
-    nim_batch_size: int = 2
-    workers_per_progress_engine: int = 5
-
-    @model_validator(mode="before")
-    @classmethod
-    def validate_endpoints(cls, values):
-        """
-        Validates the gRPC and HTTP services for all endpoints.
-
-        Ensures that at least one service (either gRPC or HTTP) is provided
-        for each endpoint in the configuration.
-
-        Parameters
-        ----------
-        values : dict
-            Dictionary containing the values of the attributes for the class.
-
-        Returns
-        -------
-        dict
-            The validated dictionary of values.
-
-        Raises
-        ------
-        ValueError
-            If both gRPC and HTTP services are empty for any endpoint.
-        """
-
-        def clean_service(service):
-            """Set service to None if it's an empty string or contains only spaces or quotes."""
-            if service is None or not service.strip() or service.strip(" \"'") == "":
-                return None
-            return service
-
-        for endpoint_name in ["yolox_endpoints", "paddle_endpoints"]:
-            grpc_service, http_service = values.get(endpoint_name, (None, None))
-            grpc_service = clean_service(grpc_service)
-            http_service = clean_service(http_service)
-
-            if not grpc_service and not http_service:
-                raise ValueError(f"Both gRPC and HTTP services cannot be empty for {endpoint_name}.")
-
-            values[endpoint_name] = (grpc_service, http_service)
-
-        return values
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ChartExtractorSchema(BaseModel):
-    """
-    Configuration schema for chart extraction processing settings.
-
-    Parameters
-    ----------
-    max_queue_size : int, default=1
-        The maximum number of items allowed in the processing queue.
-
-    n_workers : int, default=2
-        The number of worker threads to use for processing.
-
-    raise_on_failure : bool, default=False
-        A flag indicating whether to raise an exception if a failure occurs during chart extraction.
-
-    extraction_config: Optional[ChartExtractorConfigSchema], default=None
-        Configuration for the chart extraction stage, including yolox and paddle service endpoints.
-    """
-
-    max_queue_size: int = 1
-    n_workers: int = 2
-    raise_on_failure: bool = False
-
-    endpoint_config: Optional[ChartExtractorConfigSchema] = None
-
-    @field_validator("max_queue_size", "n_workers")
-    def check_positive(cls, v, field):
-        if v <= 0:
-            raise ValueError(f"{field.field_name} must be greater than 10.")
-        return v
-
-    model_config = ConfigDict(extra="forbid")

nv_ingest_api/internal/schemas/extract/extract_docx_schema.py

@@ -1,124 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
-# All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-
-import logging
-from typing import Optional
-from typing import Tuple
-
-from pydantic import model_validator, ConfigDict, BaseModel
-
-logger = logging.getLogger(__name__)
-
-
-class DocxConfigSchema(BaseModel):
-    """
-    Configuration schema for docx extraction endpoints and options.
-
-    Parameters
-    ----------
-    auth_token : Optional[str], default=None
-        Authentication token required for secure services.
-
-    yolox_endpoints : Tuple[str, str]
-        A tuple containing the gRPC and HTTP services for the yolox endpoint.
-        Either the gRPC or HTTP service can be empty, but not both.
-
-    Methods
-    -------
-    validate_endpoints(values)
-        Validates that at least one of the gRPC or HTTP services is provided for each endpoint.
-
-    Raises
-    ------
-    ValueError
-        If both gRPC and HTTP services are empty for any endpoint.
-
-    Config
-    ------
-    extra : str
-        Pydantic config option to forbid extra fields.
-    """
-
-    auth_token: Optional[str] = None
-
-    yolox_endpoints: Tuple[Optional[str], Optional[str]] = (None, None)
-    yolox_infer_protocol: str = ""
-
-    @model_validator(mode="before")
-    @classmethod
-    def validate_endpoints(cls, values):
-        """
-        Validates the gRPC and HTTP services for all endpoints.
-
-        Parameters
-        ----------
-        values : dict
-            Dictionary containing the values of the attributes for the class.
-
-        Returns
-        -------
-        dict
-            The validated dictionary of values.
-
-        Raises
-        ------
-        ValueError
-            If both gRPC and HTTP services are empty for any endpoint.
-        """
-
-        def clean_service(service):
-            """Set service to None if it's an empty string or contains only spaces or quotes."""
-            if service is None or not service.strip() or service.strip(" \"'") == "":
-                return None
-            return service
-
-        for model_name in ["yolox"]:
-            endpoint_name = f"{model_name}_endpoints"
-            grpc_service, http_service = values.get(endpoint_name)
-            grpc_service = clean_service(grpc_service)
-            http_service = clean_service(http_service)
-
-            if not grpc_service and not http_service:
-                raise ValueError(f"Both gRPC and HTTP services cannot be empty for {endpoint_name}.")
-
-            values[endpoint_name] = (grpc_service, http_service)
-
-            protocol_name = f"{model_name}_infer_protocol"
-            protocol_value = values.get(protocol_name)
-            if not protocol_value:
-                protocol_value = "http" if http_service else "grpc" if grpc_service else ""
-            protocol_value = protocol_value.lower()
-            values[protocol_name] = protocol_value
-
-        return values
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DocxExtractorSchema(BaseModel):
-    """
-    Configuration schema for the PDF extractor settings.
-
-    Parameters
-    ----------
-    max_queue_size : int, default=1
-        The maximum number of items allowed in the processing queue.
-
-    n_workers : int, default=16
-        The number of worker threads to use for processing.
-
-    raise_on_failure : bool, default=False
-        A flag indicating whether to raise an exception on processing failure.
-
-    image_extraction_config: Optional[ImageConfigSchema], default=None
-        Configuration schema for the image extraction stage.
-    """
-
-    max_queue_size: int = 1
-    n_workers: int = 16
-    raise_on_failure: bool = False
-
-    docx_extraction_config: Optional[DocxConfigSchema] = None
-    model_config = ConfigDict(extra="forbid")