nv-ingest-api 2025.6.24.dev20250625__tar.gz → 2025.7.8.dev20250708__tar.gz

{nv_ingest_api-2025.6.24.dev20250625/src/nv_ingest_api.egg-info → nv_ingest_api-2025.7.8.dev20250708}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-api
-Version: 2025.6.24.dev20250625
+Version: 2025.7.8.dev20250708
 Summary: Python module with core document ingestion functions.
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest_api-2025.6.24.dev20250625 → nv_ingest_api-2025.7.8.dev20250708}/src/nv_ingest_api/interface/utility.py

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import base64
-import os
 from io import BytesIO
 
 import pandas as pd
@@ -33,21 +32,6 @@ DOCUMENT_TO_CONTENT_MAPPING = {
 }
 
 
-# ------------------------------------------------------------------------------
-# Helper function to get the document type from a file extension.
-# ------------------------------------------------------------------------------
-def get_document_type_from_extension(file_path: str) -> str:
-    ext = os.path.splitext(file_path)[1].lower()
-    mapping = {
-        ".png": DocumentTypeEnum.PNG,
-        ".jpg": DocumentTypeEnum.JPEG,
-        ".jpeg": DocumentTypeEnum.JPEG,
-        ".tiff": DocumentTypeEnum.TIFF,
-        ".svg": DocumentTypeEnum.SVG,
-    }
-    return mapping.get(ext, DocumentTypeEnum.UNKNOWN)
-
-
 # ------------------------------------------------------------------------------
 # Helper function to read a file and return its base64-encoded string.
 # ------------------------------------------------------------------------------
@@ -99,6 +83,7 @@ def create_source_metadata(source_name: str, source_id: str, document_type: str)
         "summary": "",
         "partition_id": -1,
         "access_level": "unknown",  # You may wish to adjust this if needed.
+        "custom_content": {},
     }
 
 
@@ -130,6 +115,7 @@ def create_content_metadata(document_type: str) -> dict:
             },
         },
         "subtype": "",
+        "custom_content": {},
     }
 
 

{nv_ingest_api-2025.6.24.dev20250625 → nv_ingest_api-2025.7.8.dev20250708}/src/nv_ingest_api/internal/schemas/meta/metadata_schema.py

@@ -46,6 +46,7 @@ class SourceMetadataSchema(BaseModelNoExt):
     summary: str = ""
     partition_id: int = -1
     access_level: Union[AccessLevelEnum, int] = AccessLevelEnum.UNKNOWN
+    custom_content: Optional[Dict[str, Any]] = None
 
     @field_validator("date_created", "last_modified")
     @classmethod
@@ -99,6 +100,7 @@ class ContentMetadataSchema(BaseModelNoExt):
     subtype: Union[ContentTypeEnum, str] = ""
     start_time: int = -1
     end_time: int = -1
+    custom_content: Optional[Dict[str, Any]] = None
 
 
 class TextMetadataSchema(BaseModelNoExt):
@@ -108,6 +110,7 @@ class TextMetadataSchema(BaseModelNoExt):
     language: LanguageEnum = "en"  # default to Unknown? Maybe do some kind of heuristic check
     text_location: tuple = (0, 0, 0, 0)
     text_location_max_dimensions: tuple = (0, 0, 0, 0)
+    custom_content: Optional[Dict[str, Any]] = None
 
 
 class ImageMetadataSchema(BaseModelNoExt):
@@ -120,6 +123,7 @@ class ImageMetadataSchema(BaseModelNoExt):
     uploaded_image_url: str = ""
     width: int = 0
     height: int = 0
+    custom_content: Optional[Dict[str, Any]] = None
 
     @field_validator("image_type")
     def validate_image_type(cls, v):
@@ -143,6 +147,7 @@ class TableMetadataSchema(BaseModelNoExt):
     table_location: tuple = (0, 0, 0, 0)
     table_location_max_dimensions: tuple = (0, 0)
     uploaded_image_uri: str = ""
+    custom_content: Optional[Dict[str, Any]] = None
 
 
 class ChartMetadataSchema(BaseModelNoExt):
@@ -153,11 +158,13 @@ class ChartMetadataSchema(BaseModelNoExt):
     table_location: tuple = (0, 0, 0, 0)
     table_location_max_dimensions: tuple = (0, 0)
     uploaded_image_uri: str = ""
+    custom_content: Optional[Dict[str, Any]] = None
 
 
 class AudioMetadataSchema(BaseModelNoExt):
     audio_transcript: str = ""
     audio_type: str = ""
+    custom_content: Optional[Dict[str, Any]] = None
 
 
 # TODO consider deprecating this in favor of info msg...
@@ -166,6 +173,7 @@ class ErrorMetadataSchema(BaseModelNoExt):
     status: StatusEnum
     source_id: str = ""
     error_msg: str
+    custom_content: Optional[Dict[str, Any]] = None
 
 
 class InfoMessageMetadataSchema(BaseModelNoExt):
@@ -173,6 +181,7 @@ class InfoMessageMetadataSchema(BaseModelNoExt):
     status: StatusEnum
     message: str
     filter: bool
+    custom_content: Optional[Dict[str, Any]] = None
 
 
 # Main metadata schema
@@ -191,6 +200,7 @@ class MetadataSchema(BaseModelNoExt):
     info_message_metadata: Optional[InfoMessageMetadataSchema] = None
     debug_metadata: Optional[Dict[str, Any]] = None
     raise_on_failure: bool = False
+    custom_content: Optional[Dict[str, Any]] = None
 
     @model_validator(mode="before")
     @classmethod

{nv_ingest_api-2025.6.24.dev20250625 → nv_ingest_api-2025.7.8.dev20250708}/src/nv_ingest_api/internal/schemas/transform/transform_image_caption_schema.py

@@ -8,8 +8,8 @@ from pydantic import ConfigDict, BaseModel
 
 class ImageCaptionExtractionSchema(BaseModel):
     api_key: str = "api_key"
-    endpoint_url: str = "https://ai.api.nvidia.com/v1/gr/meta/llama-3.2-11b-vision-instruct/chat/completions"
+    endpoint_url: str = "https://ai.api.nvidia.com/v1/gr/nvidia/llama-3.1-nemotron-nano-vl-8b-v1/chat/completions"
     prompt: str = "Caption the content of this image:"
-    model_name: str = "meta/llama-3.2-11b-vision-instruct"
+    model_name: str = "nvidia/llama-3.1-nemotron-nano-vl-8b-v1"
     raise_on_failure: bool = False
     model_config = ConfigDict(extra="forbid")

{nv_ingest_api-2025.6.24.dev20250625 → nv_ingest_api-2025.7.8.dev20250708}/src/nv_ingest_api/internal/transform/embed_text.py

@@ -466,7 +466,7 @@ def transform_create_text_embeddings_internal(
 
     for content_type, content_getter in pandas_content_extractor.items():
         if not content_getter:
-            logger.debug(f"Skipping unsupported content type: {content_type}")
+            logger.warning(f"Skipping text_embedding generation for unsupported content type: {content_type}")
             continue
 
         # Get rows matching the content type

nv_ingest_api-2025.7.8.dev20250708/src/nv_ingest_api/util/imports/callable_signatures.py

@@ -0,0 +1,50 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+
+import inspect
+from pydantic import BaseModel
+
+from nv_ingest_api.internal.primitives.ingest_control_message import IngestControlMessage
+
+
+def ingest_stage_callable_signature(sig: inspect.Signature):
+    """
+    Validates that a callable has the signature:
+        (IngestControlMessage, BaseModel) -> IngestControlMessage
+
+    Raises
+    ------
+    TypeError
+        If the signature does not match the expected pattern.
+    """
+    params = list(sig.parameters.values())
+
+    if len(params) != 2:
+        raise TypeError(f"Expected exactly 2 parameters, got {len(params)}")
+
+    if params[0].name != "control_message" or params[1].name != "stage_config":
+        raise TypeError("Expected parameter names: 'control_message', 'config'")
+
+    first_param = params[0].annotation
+    second_param = params[1].annotation
+    return_type = sig.return_annotation
+
+    if first_param is inspect.Parameter.empty:
+        raise TypeError("First parameter must be annotated with IngestControlMessage")
+
+    if second_param is inspect.Parameter.empty:
+        raise TypeError("Second parameter must be annotated with a subclass of BaseModel")
+
+    if return_type is inspect.Signature.empty:
+        raise TypeError("Return type must be annotated with IngestControlMessage")
+
+    if not issubclass(first_param, IngestControlMessage):
+        raise TypeError(f"First parameter must be IngestControlMessage, got {first_param}")
+
+    if not (issubclass(second_param, BaseModel)):
+        raise TypeError(f"Second parameter must be a subclass of BaseModel, got {second_param}")
+
+    if not issubclass(return_type, IngestControlMessage):
+        raise TypeError(f"Return type must be IngestControlMessage, got {return_type}")

nv_ingest_api-2025.7.8.dev20250708/src/nv_ingest_api/util/imports/dynamic_resolvers.py

@@ -0,0 +1,110 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import importlib
+import inspect
+from typing import Callable, Union, List, Optional
+
+
+def resolve_obj_from_path(path: str, allowed_base_paths: Optional[List[str]] = None) -> object:
+    """
+    Import and return an object from a string path of the form 'module.sub:attr'.
+
+    To enhance security, this function can restrict imports to a list of allowed base module paths.
+    """
+    if ":" not in path:
+        raise ValueError(f"Invalid path '{path}': expected format 'module.sub:attr'")
+    module_path, attr_name = path.split(":", 1)
+
+    # Security check: only allow imports from specified base paths if provided.
+    if allowed_base_paths:
+        is_allowed = any(module_path == base or module_path.startswith(base + ".") for base in allowed_base_paths)
+        if not is_allowed:
+            raise ImportError(
+                f"Module '{module_path}' is not in the list of allowed base paths. "
+                f"Allowed paths: {allowed_base_paths}"
+            )
+
+    try:
+        mod = importlib.import_module(module_path)
+    except ModuleNotFoundError as e:
+        raise ImportError(f"Could not import module '{module_path}'") from e
+    try:
+        obj = getattr(mod, attr_name)
+    except AttributeError as e:
+        raise AttributeError(f"Module '{module_path}' has no attribute '{attr_name}'") from e
+    return obj
+
+
+def resolve_callable_from_path(
+    callable_path: str,
+    signature_schema: Union[List[str], Callable[[inspect.Signature], None], str],
+    allowed_base_paths: Optional[List[str]] = None,
+) -> Callable:
+    """
+    Import and return a callable from a module path string like 'module.submodule:callable_name',
+    and validate its signature using the required signature_schema (callable or path to callable).
+
+    Parameters
+    ----------
+    callable_path : str
+        The module path and callable in the format 'module.sub:callable'.
+    signature_schema : Union[List[str], Callable, str]
+        Either:
+            - A list of parameter names to require.
+            - A callable that takes an inspect.Signature and raises on failure.
+            - A string path to such a callable ('module.sub:schema_checker').
+    allowed_base_paths : Optional[List[str]]
+        An optional list of base module paths from which imports are allowed.
+        If provided, both the callable and any signature schema specified by path
+        must reside within one of these paths.
+
+    Returns
+    -------
+    Callable
+        The resolved and validated callable.
+
+    Raises
+    ------
+    ValueError
+        If the path is not correctly formatted.
+    ImportError
+        If the module cannot be imported or is not in the allowed paths.
+    AttributeError
+        If the attribute does not exist in the module.
+    TypeError
+        If the resolved attribute is not callable or the signature does not match.
+    """
+    obj = resolve_obj_from_path(callable_path, allowed_base_paths=allowed_base_paths)
+    if not callable(obj):
+        raise TypeError(f"Object '{callable_path}' is not callable")
+
+    # Load/check signature_schema
+    schema_checker = signature_schema
+    if isinstance(signature_schema, str):
+        # When loading the schema checker, apply the same security restrictions.
+        schema_checker = resolve_obj_from_path(signature_schema, allowed_base_paths=allowed_base_paths)
+
+    sig = inspect.signature(obj)
+    if isinstance(schema_checker, list):
+        actual_params = list(sig.parameters.keys())
+        missing = [p for p in schema_checker if p not in actual_params]
+        if missing:
+            raise TypeError(
+                f"Callable at '{callable_path}' is missing required parameters: {missing}\n"
+                f"Actual parameters: {actual_params}"
+            )
+    elif callable(schema_checker):
+        try:
+            schema_checker(sig)
+        except Exception as e:
+            raise TypeError(
+                f"Callable at '{callable_path}' failed custom signature validation:\n"
+                f"  Signature: {sig}\n"
+                f"  Error: {e}"
+            ) from e
+    else:
+        raise TypeError(f"Invalid signature_schema: expected list, callable, or str, got {type(signature_schema)}")
+
+    return obj

nv_ingest_api-2025.7.8.dev20250708/src/nv_ingest_api/util/service_clients/redis/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0

{nv_ingest_api-2025.6.24.dev20250625 → nv_ingest_api-2025.7.8.dev20250708/src/nv_ingest_api.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-api
-Version: 2025.6.24.dev20250625
+Version: 2025.7.8.dev20250708
 Summary: Python module with core document ingestion functions.
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest_api-2025.6.24.dev20250625 → nv_ingest_api-2025.7.8.dev20250708}/src/nv_ingest_api.egg-info/SOURCES.txt

@@ -134,6 +134,9 @@ src/nv_ingest_api/util/image_processing/clustering.py
 src/nv_ingest_api/util/image_processing/processing.py
 src/nv_ingest_api/util/image_processing/table_and_chart.py
 src/nv_ingest_api/util/image_processing/transforms.py
+src/nv_ingest_api/util/imports/__init__.py
+src/nv_ingest_api/util/imports/callable_signatures.py
+src/nv_ingest_api/util/imports/dynamic_resolvers.py
 src/nv_ingest_api/util/logging/__init__.py
 src/nv_ingest_api/util/logging/configuration.py
 src/nv_ingest_api/util/message_brokers/__init__.py