nv-ingest-api 2025.4.17.dev20250417__py3-none-any.whl → 2025.4.19.dev20250419__py3-none-any.whl

nv_ingest_api/internal/extract/pptx/pptx_extractor.py

@@ -1,187 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
-# All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-import base64
-import functools
-import io
-import logging
-from typing import Any, Optional, Dict, Union, Tuple
-
-import pandas as pd
-from pydantic import BaseModel
-
-from nv_ingest_api.internal.extract.pptx.engines.pptx_helper import python_pptx
-from nv_ingest_api.util.exception_handlers.decorators import unified_exception_handler
-
-logger = logging.getLogger(__name__)
-
-
-def _prepare_task_properties(
-    base64_row: pd.Series, task_props: Union[Dict[str, Any], BaseModel]
-) -> Tuple[Dict[str, Any], Optional[str]]:
-    """
-    Prepare and return the task properties dictionary and source identifier from a DataFrame row.
-
-    This function converts task properties to a dictionary (if provided as a Pydantic model),
-    extracts row data (excluding the "content" field), and stores it under the "row_data" key within
-    the task properties. It also retrieves the "source_id" from the row if present.
-
-    Parameters
-    ----------
-    base64_row : pd.Series
-        A pandas Series representing a row containing base64-encoded content under the key "content"
-        and optionally a "source_id".
-    task_props : Union[Dict[str, Any], BaseModel]
-        A dictionary or Pydantic model containing extraction instructions and parameters.
-
-    Returns
-    -------
-    Tuple[Dict[str, Any], Optional[str]]
-        A tuple where the first element is the prepared task properties dictionary (with "row_data" added)
-        and the second element is the source_id if present; otherwise, None.
-    """
-    # If task_props is a Pydantic model, convert it to a dictionary.
-    if isinstance(task_props, BaseModel):
-        task_props = task_props.model_dump()
-    else:
-        task_props = dict(task_props)
-
-    # Exclude the "content" field from the row data.
-    row_data = base64_row.drop(labels=["content"], errors="ignore")
-    if "params" not in task_props:
-        task_props["params"] = {}
-    # Store the row data in the parameters.
-    task_props["params"]["row_data"] = row_data
-
-    # Retrieve the source identifier if available.
-    source_id = base64_row.get("source_id", None)
-    return task_props, source_id
-
-
-@unified_exception_handler
-def _decode_and_extract_from_pptx(
-    base64_row: pd.Series,
-    task_props: Union[Dict[str, Any], BaseModel],
-    extraction_config: Any,
-    trace_info: Dict[str, Any],
-) -> Any:
-    """
-    Decode base64-encoded PPTX content from a DataFrame row and extract data using the specified method.
-
-    The function prepares task properties (using `_prepare_task_properties`), decodes the base64 content
-    into a byte stream, determines extraction parameters, and calls the extraction function (e.g. `python_pptx`)
-    with the proper flags. If extraction fails, an exception tag is returned.
-
-    Parameters
-    ----------
-    base64_row : pd.Series
-        A Series containing base64-encoded PPTX content under the key "content" and optionally a "source_id".
-    task_props : Union[Dict[str, Any], BaseModel]
-        A dictionary or Pydantic model containing extraction instructions (may include a "method" key and "params").
-    extraction_config : Any
-        A configuration object containing PPTX extraction settings (e.g. `pptx_extraction_config`).
-    trace_info : Dict[str, Any]
-        A dictionary with trace information for logging or debugging.
-
-    Returns
-    -------
-    Any
-        The extracted data from the PPTX file, or an exception tag indicating failure.
-    """
-    # Prepare task properties and extract source_id.
-    prepared_task_props, source_id = _prepare_task_properties(base64_row, task_props)
-
-    # Decode base64 content into bytes and create a BytesIO stream.
-    base64_content: str = base64_row["content"]
-    pptx_bytes: bytes = base64.b64decode(base64_content)
-    pptx_stream: io.BytesIO = io.BytesIO(pptx_bytes)
-
-    # Retrieve extraction parameters (and remove boolean flags as they are consumed).
-    extract_params: Dict[str, Any] = prepared_task_props.get("params", {})
-    try:
-        extract_text: bool = extract_params.pop("extract_text", False)
-        extract_images: bool = extract_params.pop("extract_images", False)
-        extract_tables: bool = extract_params.pop("extract_tables", False)
-        extract_charts: bool = extract_params.pop("extract_charts", False)
-        extract_infographics: bool = extract_params.pop("extract_infographics", False)
-    except KeyError as e:
-        raise ValueError(f"Missing required extraction flag: {e}")
-
-    # Inject additional configuration and trace information.
-    if getattr(extraction_config, "pptx_extraction_config", None) is not None:
-        extract_params["pptx_extraction_config"] = extraction_config.pptx_extraction_config
-    if trace_info is not None:
-        extract_params["trace_info"] = trace_info
-
-    # Call the PPTX extraction function.
-    extracted_data = python_pptx(
-        pptx_stream=pptx_stream,
-        extract_text=extract_text,
-        extract_images=extract_images,
-        extract_infographics=extract_infographics,
-        extract_tables=extract_tables,
-        extract_charts=extract_charts,
-        extraction_config=extract_params,
-        execution_trace_log=None,
-    )
-
-    return extracted_data
-
-
-@unified_exception_handler
-def extract_primitives_from_pptx_internal(
-    df_extraction_ledger: pd.DataFrame,
-    task_config: Union[Dict[str, Any], BaseModel],
-    extraction_config: Any,  # Assuming PPTXExtractorSchema or similar type
-    execution_trace_log: Optional[Dict[str, Any]] = None,
-) -> pd.DataFrame:
-    """
-    Process a DataFrame containing base64-encoded PPTX files and extract primitive data.
-
-    This function applies a decoding and extraction routine to each row of the DataFrame
-    (via `_decode_and_extract_from_pptx`), then explodes any list results into separate rows, drops missing values,
-    and compiles the extracted data into a new DataFrame. The resulting DataFrame includes columns for document type,
-    extracted metadata, and a unique identifier (UUID).
-
-    Parameters
-    ----------
-    df_extraction_ledger : pd.DataFrame
-        Input DataFrame with PPTX files in base64 encoding. Expected to include columns 'source_id' and 'content'.
-    task_config : Union[Dict[str, Any], BaseModel]
-        Configuration for the PPTX extraction task, as a dict or Pydantic model.
-    extraction_config : Any
-        Configuration object for PPTX extraction (e.g. PPTXExtractorSchema).
-    execution_trace_log : Optional[Dict[str, Any]], optional
-        Optional dictionary containing trace information for debugging.
-
-    Returns
-    -------
-    pd.DataFrame
-        DataFrame with extracted PPTX content containing columns:
-        "document_type", "metadata", and "uuid".
-
-    Raises
-    ------
-    Exception
-        Reraises any exception encountered during extraction with additional context.
-    """
-    # Create a partial function to decode and extract content from each DataFrame row.
-    decode_and_extract_partial = functools.partial(
-        _decode_and_extract_from_pptx,
-        task_props=task_config,
-        extraction_config=extraction_config,
-        trace_info=execution_trace_log,
-    )
-    # Apply the decoding and extraction to each row.
-    extraction_series = df_extraction_ledger.apply(decode_and_extract_partial, axis=1)
-    # Explode list results into separate rows and remove missing values.
-    extraction_series = extraction_series.explode().dropna()
-
-    # Convert the series into a DataFrame with defined columns.
-    if not extraction_series.empty:
-        extracted_df = pd.DataFrame(extraction_series.to_list(), columns=["document_type", "metadata", "uuid"])
-    else:
-        extracted_df = pd.DataFrame({"document_type": [], "metadata": [], "uuid": []})
-
-    return extracted_df

nv_ingest_api/internal/mutate/__init__.py

@@ -1,3 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
-# All rights reserved.
-# SPDX-License-Identifier: Apache-2.0

nv_ingest_api/internal/mutate/deduplicate.py

@@ -1,110 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
-# All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-import logging
-import hashlib
-from typing import Any, Dict, Optional, List
-
-import pandas as pd
-
-from nv_ingest_api.internal.enums.common import ContentTypeEnum
-from nv_ingest_api.internal.schemas.mutate.mutate_image_dedup_schema import ImageDedupSchema
-
-logger = logging.getLogger(__name__)
-
-
-def _hash_content(x: Any, algorithm: str = "md5") -> bytes:
-    """
-    Compute a hash of the content using the specified algorithm.
-
-    Parameters
-    ----------
-    x : dict
-        A dictionary containing the content under the key "content".
-    algorithm : str, optional
-        Hashing algorithm to use (default "md5").
-
-    Returns
-    -------
-    bytes
-        The computed hash.
-    """
-    try:
-        return hashlib.new(algorithm, x["content"].encode()).digest()
-    except Exception as e:
-        msg = f"hash_content: Error computing hash: {e}"
-        logger.error(msg, exc_info=True)
-        raise type(e)(msg) from e
-
-
-def deduplicate_images_internal(
-    df_ledger: pd.DataFrame,
-    task_config: Dict[str, Any],
-    mutate_config: ImageDedupSchema = ImageDedupSchema(),
-    execution_trace_log: Optional[List[Any]] = None,
-) -> pd.DataFrame:
-    """
-    Deduplicate images in a DataFrame based on content hashes.
-
-    The function processes rows where the 'document_type' is IMAGE, computes a content hash for each,
-    and then either removes duplicates or marks them based on the 'filter' flag in task_config.
-    A 'hash_algorithm' flag in task_config determines the algorithm used for hashing.
-
-    Parameters
-    ----------
-    df_ledger : pd.DataFrame
-        DataFrame containing at least 'document_type' and 'metadata' columns.
-    task_config : dict
-        Configuration parameters, including:
-            - "filter": bool, if True duplicate rows are removed; if False, duplicates are marked.
-            - "hash_algorithm": str, the algorithm to use for hashing (default "md5").
-    mutate_config : ImageDedupSchema, optional
-    execution_trace_log : Optional[List[Any]], optional
-
-    Returns
-    -------
-    pd.DataFrame
-        The DataFrame with duplicate images either removed or marked.
-
-    Raises
-    ------
-    ValueError
-        If the required columns are missing.
-    Exception
-        For any other errors encountered during deduplication.
-    """
-
-    _ = mutate_config  # Unused variable
-    _ = execution_trace_log  # TODO(Devin): Implement trace logging
-
-    try:
-        # Verify required columns exist.
-        for col in ("document_type", "metadata"):
-            if col not in df_ledger.columns:
-                raise ValueError(f"Missing required column '{col}'.")
-
-        # Select image rows.
-        image_mask = df_ledger["document_type"] == ContentTypeEnum.IMAGE
-        if not image_mask.any():
-            return df_ledger[~image_mask]
-
-        df_images = df_ledger.loc[image_mask].copy()
-        hash_algorithm = task_config.get("hash_algorithm", "md5")
-
-        # Compute content hash for each image.
-        df_images["_image_content_hash"] = df_images["metadata"].apply(_hash_content, args=(hash_algorithm,))
-        df_images_deduped = df_images.drop_duplicates(subset="_image_content_hash")
-        deduped_indices = df_images_deduped.index
-
-        non_image_rows = df_ledger.loc[~image_mask]
-        deduped_images = df_images.loc[deduped_indices][df_ledger.columns.difference(["_image_content_hash"])]
-
-        result, execution_trace_log = pd.concat([deduped_images, non_image_rows], axis=0), {}
-        _ = execution_trace_log
-
-        return result
-    except Exception as e:
-        msg = f"deduplicate_images_internal: Error applying deduplication filter: {e}"
-        logger.error(msg, exc_info=True)
-        raise type(e)(msg) from e

nv_ingest_api/internal/mutate/filter.py

@@ -1,133 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
-# All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-import logging
-from typing import Dict, Optional, List, Any
-
-import pandas as pd
-
-from nv_ingest_api.internal.enums.common import TaskTypeEnum
-from nv_ingest_api.internal.schemas.meta.metadata_schema import (
-    ContentTypeEnum,
-    InfoMessageMetadataSchema,
-    StatusEnum,
-)
-from nv_ingest_api.internal.schemas.transform.transform_image_filter_schema import ImageFilterSchema
-from nv_ingest_api.util.schema.schema_validator import validate_schema
-
-logger = logging.getLogger(__name__)
-
-
-def _add_info_message(x, info_msg):
-    x["info_message_metadata"] = info_msg
-
-    return x
-
-
-def _calculate_average_image_size(x):
-    return (x["image_metadata"]["width"] + x["image_metadata"]["height"]) / 2
-
-
-def _calculate_aspect_ratio(x):
-    return x["image_metadata"]["width"] / max(x["image_metadata"]["height"], 1e-9)
-
-
-def filter_images_internal(
-    df_ledger: pd.DataFrame,
-    task_config: Dict[str, Any],
-    mutate_config: ImageFilterSchema = ImageFilterSchema(),
-    execution_trace_log: Optional[List[Any]] = None,
-) -> pd.DataFrame:
-    """
-    Apply an image filtering operation to a DataFrame based on average image size and aspect ratio.
-
-    Parameters
-    ----------
-    df_ledger : pd.DataFrame
-        DataFrame to be filtered. Must contain 'document_type' and 'metadata' columns.
-    task_config : dict
-        Dictionary with the following keys:
-            - "min_size": Minimum average image size threshold.
-            - "max_aspect_ratio": Maximum allowed aspect ratio.
-            - "min_aspect_ratio": Minimum allowed aspect ratio.
-            - "filter": If True, rows failing the criteria are dropped; if False, they are flagged.
-    mutate_config : ImageFilterSchema
-    execution_trace_log : Optional[List[Any]], optional
-
-    Returns
-    -------
-    pd.DataFrame
-        The updated DataFrame after applying the image filter.
-
-    Raises
-    ------
-    ValueError
-        If required columns are missing or if parameters are invalid.
-    Exception
-        For other errors encountered during filtering.
-    """
-
-    _ = mutate_config  # Unused variable
-    _ = execution_trace_log  # TODO(Devin)
-
-    try:
-        required_columns = {"document_type", "metadata"}
-        if not required_columns.issubset(df_ledger.columns):
-            raise ValueError(f"DataFrame must contain columns: {required_columns}")
-
-        min_size = task_config.get("min_size")
-        max_aspect_ratio = task_config.get("max_aspect_ratio")
-        min_aspect_ratio = task_config.get("min_aspect_ratio")
-        filter_flag = task_config.get("filter", True)
-
-        if not isinstance(min_size, (int, float)) or min_size < 0:
-            raise ValueError("min_size must be a non-negative number")
-        if not isinstance(max_aspect_ratio, (int, float)) or max_aspect_ratio <= 0:
-            raise ValueError("max_aspect_ratio must be a positive number")
-        if not isinstance(min_aspect_ratio, (int, float)) or min_aspect_ratio <= 0:
-            raise ValueError("min_aspect_ratio must be a positive number")
-        if min_aspect_ratio > max_aspect_ratio:
-            raise ValueError("min_aspect_ratio cannot be greater than max_aspect_ratio")
-
-        image_mask = df_ledger["document_type"] == ContentTypeEnum.IMAGE
-        if not image_mask.any():
-            return df_ledger.copy()
-
-        df_image = df_ledger.loc[image_mask].copy()
-        avg_size = df_image["metadata"].apply(_calculate_average_image_size)
-        avg_size_mask = avg_size > min_size
-
-        aspect_ratio = df_image["metadata"].apply(_calculate_aspect_ratio)
-        min_aspect_ratio_mask = aspect_ratio > min_aspect_ratio
-        max_aspect_ratio_mask = aspect_ratio < max_aspect_ratio
-
-        valid_mask = avg_size_mask & min_aspect_ratio_mask & max_aspect_ratio_mask
-        image_filter_mask = ~valid_mask
-
-        if image_filter_mask.any():
-            filtered_df = df_image.loc[image_filter_mask].copy()
-            if filter_flag:
-                df_ledger.drop(labels=filtered_df.index, inplace=True)
-                return df_ledger
-
-            info_msg = {
-                "task": TaskTypeEnum.FILTER.value,
-                "status": StatusEnum.SUCCESS.value,
-                "message": "Filtered due to image size or aspect ratio.",
-                "filter": True,
-            }
-            validated_info_msg = validate_schema(info_msg, InfoMessageMetadataSchema).model_dump()
-            filtered_df["info_message_metadata"] = [validated_info_msg] * filtered_df.shape[0]
-            filtered_df["metadata"] = filtered_df["metadata"].apply(_add_info_message, args=(info_msg,))
-            df_ledger.loc[filtered_df.index, "metadata"] = filtered_df["metadata"]
-            df_ledger.loc[filtered_df.index, "document_type"] = ContentTypeEnum.INFO_MSG
-
-        result, execution_trace_log = df_ledger, {}
-
-        return result
-
-    except Exception as e:
-        err_msg = f"filter_images_internal: Error applying image filter. Original error: {e}"
-        logger.error(err_msg, exc_info=True)
-        raise type(e)(err_msg) from e

nv_ingest_api/internal/primitives/nim/__init__.py

@@ -1,8 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
-# All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-from .nim_client import NimClient
-from .nim_model_interface import ModelInterface
-
-__all__ = ["NimClient", "ModelInterface"]

nv_ingest_api/internal/primitives/nim/default_values.py

@@ -1,15 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
-# All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-# Copyright (c) 2024, NVIDIA CORPORATION.
-
-
-YOLOX_MAX_BATCH_SIZE = 8
-YOLOX_MAX_WIDTH = 1536
-YOLOX_MAX_HEIGHT = 1536
-YOLOX_NUM_CLASSES = 3
-YOLOX_CONF_THRESHOLD = 0.01
-YOLOX_IOU_THRESHOLD = 0.5
-YOLOX_MIN_SCORE = 0.1
-YOLOX_FINAL_SCORE = 0.48

nv_ingest_api/internal/primitives/nim/model_interface/__init__.py

@@ -1,3 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
-# All rights reserved.
-# SPDX-License-Identifier: Apache-2.0