nv-ingest-api 2025.10.26.dev20251026__tar.gz → 2025.10.28.dev20251028__tar.gz

{nv_ingest_api-2025.10.26.dev20251026/src/nv_ingest_api.egg-info → nv_ingest_api-2025.10.28.dev20251028}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-api
-Version: 2025.10.26.dev20251026
+Version: 2025.10.28.dev20251028
 Summary: Python module with core document ingestion functions.
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License
@@ -222,6 +222,7 @@ Requires-Dist: fsspec>=2025.5.1
 Requires-Dist: universal_pathlib>=0.2.6
 Requires-Dist: ffmpeg-python==0.2.0
 Requires-Dist: tritonclient
+Requires-Dist: glom
 Dynamic: license-file
 
 # nv-ingest-api

{nv_ingest_api-2025.10.26.dev20251026 → nv_ingest_api-2025.10.28.dev20251028}/pyproject.toml

@@ -29,6 +29,7 @@ dependencies = [
     "universal_pathlib>=0.2.6",
     "ffmpeg-python==0.2.0",
     "tritonclient",
+    "glom",
 ]
 
 [project.urls]

{nv_ingest_api-2025.10.26.dev20251026 → nv_ingest_api-2025.10.28.dev20251028}/src/nv_ingest_api/internal/primitives/nim/model_interface/parakeet.py

@@ -355,6 +355,10 @@ def create_audio_inference_client(
     if (infer_protocol is None) and (grpc_endpoint and grpc_endpoint.strip()):
         infer_protocol = "grpc"
 
+    # Normalize protocol to lowercase for case-insensitive comparison
+    if infer_protocol:
+        infer_protocol = infer_protocol.lower()
+
     if infer_protocol == "http":
         raise ValueError("`http` endpoints are not supported for audio. Use `grpc`.")
 

{nv_ingest_api-2025.10.26.dev20251026 → nv_ingest_api-2025.10.28.dev20251028}/src/nv_ingest_api/internal/schemas/extract/extract_audio_schema.py

@@ -10,10 +10,12 @@ from typing import Tuple
 from pydantic import BaseModel, Field
 from pydantic import root_validator
 
+from nv_ingest_api.internal.schemas.mixins import LowercaseProtocolMixin
+
 logger = logging.getLogger(__name__)
 
 
-class AudioConfigSchema(BaseModel):
+class AudioConfigSchema(LowercaseProtocolMixin):
     """
     Configuration schema for audio extraction endpoints and options.
 
@@ -87,13 +89,13 @@ class AudioConfigSchema(BaseModel):
 
         values[endpoint_name] = (grpc_service, http_service)
 
+        # Auto-infer protocol from endpoints if not specified
         protocol_name = "audio_infer_protocol"
         protocol_value = values.get(protocol_name)
 
         if not protocol_value:
             protocol_value = "http" if http_service else "grpc" if grpc_service else ""
 
-        protocol_value = protocol_value.lower()
         values[protocol_name] = protocol_value
 
         return values

{nv_ingest_api-2025.10.26.dev20251026 → nv_ingest_api-2025.10.28.dev20251028}/src/nv_ingest_api/internal/schemas/extract/extract_chart_schema.py

@@ -8,10 +8,12 @@ from typing import Tuple
 
 from pydantic import field_validator, model_validator, ConfigDict, BaseModel, Field
 
+from nv_ingest_api.internal.schemas.mixins import LowercaseProtocolMixin
+
 logger = logging.getLogger(__name__)
 
 
-class ChartExtractorConfigSchema(BaseModel):
+class ChartExtractorConfigSchema(LowercaseProtocolMixin):
     """
     Configuration schema for chart extraction service endpoints and options.
 
@@ -96,6 +98,13 @@ class ChartExtractorConfigSchema(BaseModel):
 
             values[endpoint_name] = (grpc_service, http_service)
 
+            # Auto-infer protocol from endpoints if not specified
+            protocol_name = endpoint_name.replace("_endpoints", "_infer_protocol")
+            protocol_value = values.get(protocol_name)
+            if not protocol_value:
+                protocol_value = "http" if http_service else "grpc" if grpc_service else ""
+            values[protocol_name] = protocol_value
+
         return values
 
     model_config = ConfigDict(extra="forbid")

{nv_ingest_api-2025.10.26.dev20251026 → nv_ingest_api-2025.10.28.dev20251028}/src/nv_ingest_api/internal/schemas/extract/extract_docx_schema.py

@@ -9,10 +9,12 @@ from typing import Tuple
 
 from pydantic import model_validator, ConfigDict, BaseModel, Field
 
+from nv_ingest_api.internal.schemas.mixins import LowercaseProtocolMixin
+
 logger = logging.getLogger(__name__)
 
 
-class DocxConfigSchema(BaseModel):
+class DocxConfigSchema(LowercaseProtocolMixin):
     """
     Configuration schema for docx extraction endpoints and options.
 
@@ -85,11 +87,11 @@ class DocxConfigSchema(BaseModel):
 
             values[endpoint_name] = (grpc_service, http_service)
 
+            # Auto-infer protocol from endpoints if not specified
             protocol_name = f"{model_name}_infer_protocol"
             protocol_value = values.get(protocol_name)
             if not protocol_value:
                 protocol_value = "http" if http_service else "grpc" if grpc_service else ""
-            protocol_value = protocol_value.lower()
             values[protocol_name] = protocol_value
 
         return values

{nv_ingest_api-2025.10.26.dev20251026 → nv_ingest_api-2025.10.28.dev20251028}/src/nv_ingest_api/internal/schemas/extract/extract_image_schema.py

@@ -9,10 +9,12 @@ from typing import Tuple
 
 from pydantic import model_validator, ConfigDict, BaseModel, Field
 
+from nv_ingest_api.internal.schemas.mixins import LowercaseProtocolMixin
+
 logger = logging.getLogger(__name__)
 
 
-class ImageConfigSchema(BaseModel):
+class ImageConfigSchema(LowercaseProtocolMixin):
     """
     Configuration schema for image extraction endpoints and options.
 
@@ -85,11 +87,11 @@ class ImageConfigSchema(BaseModel):
 
             values[endpoint_name] = (grpc_service, http_service)
 
+            # Auto-infer protocol from endpoints if not specified
             protocol_name = f"{model_name}_infer_protocol"
             protocol_value = values.get(protocol_name)
             if not protocol_value:
                 protocol_value = "http" if http_service else "grpc" if grpc_service else ""
-            protocol_value = protocol_value.lower()
             values[protocol_name] = protocol_value
 
         return values

{nv_ingest_api-2025.10.26.dev20251026 → nv_ingest_api-2025.10.28.dev20251028}/src/nv_ingest_api/internal/schemas/extract/extract_infographic_schema.py

@@ -8,10 +8,12 @@ from typing import Tuple
 
 from pydantic import field_validator, model_validator, ConfigDict, BaseModel, Field
 
+from nv_ingest_api.internal.schemas.mixins import LowercaseProtocolMixin
+
 logger = logging.getLogger(__name__)
 
 
-class InfographicExtractorConfigSchema(BaseModel):
+class InfographicExtractorConfigSchema(LowercaseProtocolMixin):
     """
     Configuration schema for infographic extraction service endpoints and options.
 
@@ -89,6 +91,13 @@ class InfographicExtractorConfigSchema(BaseModel):
 
             values[endpoint_name] = (grpc_service, http_service)
 
+            # Auto-infer protocol from endpoints if not specified
+            protocol_name = endpoint_name.replace("_endpoints", "_infer_protocol")
+            protocol_value = values.get(protocol_name)
+            if not protocol_value:
+                protocol_value = "http" if http_service else "grpc" if grpc_service else ""
+            values[protocol_name] = protocol_value
+
         return values
 
     model_config = ConfigDict(extra="forbid")

{nv_ingest_api-2025.10.26.dev20251026 → nv_ingest_api-2025.10.28.dev20251028}/src/nv_ingest_api/internal/schemas/extract/extract_pdf_schema.py

@@ -9,10 +9,12 @@ from typing import Tuple
 
 from pydantic import model_validator, ConfigDict, BaseModel, Field
 
+from nv_ingest_api.internal.schemas.mixins import LowercaseProtocolMixin
+
 logger = logging.getLogger(__name__)
 
 
-class PDFiumConfigSchema(BaseModel):
+class PDFiumConfigSchema(LowercaseProtocolMixin):
     """
     Configuration schema for PDFium endpoints and options.
 
@@ -82,11 +84,11 @@ class PDFiumConfigSchema(BaseModel):
 
             values[endpoint_name] = (grpc_service, http_service)
 
+            # Auto-infer protocol from endpoints if not specified
             protocol_name = f"{model_name}_infer_protocol"
             protocol_value = values.get(protocol_name)
             if not protocol_value:
                 protocol_value = "http" if http_service else "grpc" if grpc_service else ""
-            protocol_value = protocol_value.lower()
             values[protocol_name] = protocol_value
 
         return values
@@ -94,7 +96,7 @@ class PDFiumConfigSchema(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
 
-class NemoRetrieverParseConfigSchema(BaseModel):
+class NemoRetrieverParseConfigSchema(LowercaseProtocolMixin):
     """
     Configuration schema for NemoRetrieverParse endpoints and options.
 
@@ -170,11 +172,11 @@ class NemoRetrieverParseConfigSchema(BaseModel):
 
             values[endpoint_name] = (grpc_service, http_service)
 
+            # Auto-infer protocol from endpoints if not specified
             protocol_name = f"{model_name}_infer_protocol"
             protocol_value = values.get(protocol_name)
             if not protocol_value:
                 protocol_value = "http" if http_service else "grpc" if grpc_service else ""
-            protocol_value = protocol_value.lower()
             values[protocol_name] = protocol_value
 
         return values

{nv_ingest_api-2025.10.26.dev20251026 → nv_ingest_api-2025.10.28.dev20251028}/src/nv_ingest_api/internal/schemas/extract/extract_pptx_schema.py

@@ -9,10 +9,12 @@ from typing import Tuple
 
 from pydantic import model_validator, ConfigDict, BaseModel, Field
 
+from nv_ingest_api.internal.schemas.mixins import LowercaseProtocolMixin
+
 logger = logging.getLogger(__name__)
 
 
-class PPTXConfigSchema(BaseModel):
+class PPTXConfigSchema(LowercaseProtocolMixin):
     """
     Configuration schema for docx extraction endpoints and options.
 
@@ -85,11 +87,11 @@ class PPTXConfigSchema(BaseModel):
 
             values[endpoint_name] = (grpc_service, http_service)
 
+            # Auto-infer protocol from endpoints if not specified
             protocol_name = f"{model_name}_infer_protocol"
             protocol_value = values.get(protocol_name)
             if not protocol_value:
                 protocol_value = "http" if http_service else "grpc" if grpc_service else ""
-            protocol_value = protocol_value.lower()
             values[protocol_name] = protocol_value
 
         return values

{nv_ingest_api-2025.10.26.dev20251026 → nv_ingest_api-2025.10.28.dev20251028}/src/nv_ingest_api/internal/schemas/extract/extract_table_schema.py

@@ -9,11 +9,12 @@ from typing import Tuple
 
 from pydantic import field_validator, model_validator, ConfigDict, BaseModel, Field
 
+from nv_ingest_api.internal.schemas.mixins import LowercaseProtocolMixin
 
 logger = logging.getLogger(__name__)
 
 
-class TableExtractorConfigSchema(BaseModel):
+class TableExtractorConfigSchema(LowercaseProtocolMixin):
     """
     Configuration schema for the table extraction stage settings.
 
@@ -91,6 +92,13 @@ class TableExtractorConfigSchema(BaseModel):
 
             values[endpoint_name] = (grpc_service, http_service)
 
+            # Auto-infer protocol from endpoints if not specified
+            protocol_name = endpoint_name.replace("_endpoints", "_infer_protocol")
+            protocol_value = values.get(protocol_name)
+            if not protocol_value:
+                protocol_value = "http" if http_service else "grpc" if grpc_service else ""
+            values[protocol_name] = protocol_value
+
         return values
 
     model_config = ConfigDict(extra="forbid")

{nv_ingest_api-2025.10.26.dev20251026 → nv_ingest_api-2025.10.28.dev20251028}/src/nv_ingest_api/internal/schemas/meta/ingest_job_schema.py

@@ -126,6 +126,8 @@ class IngestTaskEmbedSchema(BaseModelNoExt):
     image_elements_modality: Optional[str] = None
     structured_elements_modality: Optional[str] = None
     audio_elements_modality: Optional[str] = None
+    custom_content_field: Optional[str] = None
+    result_target_field: Optional[str] = None
 
 
 class IngestTaskVdbUploadSchema(BaseModelNoExt):

nv_ingest_api-2025.10.28.dev20251028/src/nv_ingest_api/internal/schemas/mixins.py

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Shared mixins for Pydantic schemas.
+"""
+
+from typing import Any
+from pydantic import BaseModel, field_validator
+
+
+class LowercaseProtocolMixin(BaseModel):
+    """
+    Mixin that automatically lowercases any field ending with '_infer_protocol'.
+
+    This ensures case-insensitive handling of protocol values (e.g., "HTTP" -> "http").
+    Apply this mixin to any schema that has protocol fields to normalize user input.
+
+    Examples
+    --------
+    >>> class MyConfigSchema(LowercaseProtocolMixin):
+    ...     yolox_infer_protocol: str = ""
+    ...     ocr_infer_protocol: str = ""
+    >>>
+    >>> config = MyConfigSchema(yolox_infer_protocol="GRPC", ocr_infer_protocol="HTTP")
+    >>> config.yolox_infer_protocol
+    'grpc'
+    >>> config.ocr_infer_protocol
+    'http'
+    """
+
+    @field_validator("*", mode="before")
+    @classmethod
+    def _lowercase_protocol_fields(cls, v: Any, info):
+        """Lowercase any field ending with '_infer_protocol'."""
+        if info.field_name.endswith("_infer_protocol") and v is not None:
+            return str(v).strip().lower()
+        return v

{nv_ingest_api-2025.10.26.dev20251026 → nv_ingest_api-2025.10.28.dev20251028}/src/nv_ingest_api/internal/schemas/transform/transform_text_embedding_schema.py

@@ -7,6 +7,8 @@ import logging
 
 from pydantic import ConfigDict, BaseModel, Field, model_validator, field_validator
 
+from typing import Optional
+
 from nv_ingest_api.util.logging.configuration import LogLevel
 
 logger = logging.getLogger(__name__)
@@ -26,6 +28,8 @@ class TextEmbeddingSchema(BaseModel):
     image_elements_modality: str = Field(default="text")
     structured_elements_modality: str = Field(default="text")
     audio_elements_modality: str = Field(default="text")
+    custom_content_field: Optional[str] = None
+    result_target_field: Optional[str] = None
 
     model_config = ConfigDict(extra="forbid")
 

{nv_ingest_api-2025.10.26.dev20251026 → nv_ingest_api-2025.10.28.dev20251028}/src/nv_ingest_api/internal/transform/embed_text.py

@@ -7,6 +7,7 @@ from concurrent.futures import ThreadPoolExecutor
 from functools import partial
 from typing import Any, Dict, Tuple, Optional, Iterable, List
 
+import glom
 import pandas as pd
 from openai import OpenAI
 
@@ -282,6 +283,33 @@ def _add_embeddings(row, embeddings, info_msgs):
     return row
 
 
+def _add_custom_embeddings(row, embeddings, result_target_field):
+    """
+    Updates a DataFrame row with embedding data and associated error info
+    based on a user supplied custom content field.
+
+    Parameters
+    ----------
+    row : pandas.Series
+        A row of the DataFrame.
+    embeddings : dict
+        Dictionary mapping row indices to embeddings.
+    result_target_field: str
+        The field in custom_content to output the embeddings to
+
+    Returns
+    -------
+    pandas.Series
+        The updated row
+    """
+    embedding = embeddings.get(row.name, None)
+
+    if embedding is not None:
+        row["metadata"] = glom.assign(row["metadata"], "custom_content." + result_target_field, embedding, missing=dict)
+
+    return row
+
+
 def _format_image_input_string(image_b64: Optional[str]) -> str:
     if not image_b64:
         return
@@ -381,6 +409,20 @@ def _get_pandas_audio_content(row, modality="text"):
     return row.get("audio_metadata", {}).get("audio_transcript")
 
 
+def _get_pandas_custom_content(row, custom_content_field):
+    custom_content = row.get("custom_content", {})
+    content = glom.glom(custom_content, custom_content_field, default=None)
+    if content is None:
+        logger.warning(f"Custom content field: {custom_content_field} not found")
+        return None
+
+    try:
+        return str(content)
+    except (TypeError, ValueError):
+        logger.warning(f"Cannot convert custom content field: {custom_content_field} to string")
+        return None
+
+
 # ------------------------------------------------------------------------------
 # Batch Processing Utilities
 # ------------------------------------------------------------------------------
@@ -519,6 +561,7 @@ def transform_create_text_embeddings_internal(
     api_key = task_config.get("api_key") or transform_config.api_key
     endpoint_url = task_config.get("endpoint_url") or transform_config.embedding_nim_endpoint
     model_name = task_config.get("model_name") or transform_config.embedding_model
+    custom_content_field = task_config.get("custom_content_field") or transform_config.custom_content_field
 
     if execution_trace_log is None:
         execution_trace_log = {}
@@ -612,4 +655,43 @@ def transform_create_text_embeddings_internal(
         content_masks.append(content_mask)
 
     combined_df = _concatenate_extractions_pandas(df_transform_ledger, embedding_dataframes, content_masks)
+
+    # Embed custom content
+    if custom_content_field is not None:
+        result_target_field = task_config.get("result_target_field") or custom_content_field + "_embedding"
+
+        extracted_custom_content = (
+            combined_df["metadata"]
+            .apply(partial(_get_pandas_custom_content, custom_content_field=custom_content_field))
+            .apply(lambda x: x.strip() if isinstance(x, str) and x.strip() else None)
+        )
+
+        valid_custom_content_mask = extracted_custom_content.notna()
+        if valid_custom_content_mask.any():
+            custom_content_list = extracted_custom_content[valid_custom_content_mask].to_list()
+            custom_content_batches = _generate_batches(custom_content_list, batch_size=transform_config.batch_size)
+
+            custom_content_embeddings = _async_runner(
+                custom_content_batches,
+                api_key,
+                endpoint_url,
+                model_name,
+                transform_config.encoding_format,
+                transform_config.input_type,
+                transform_config.truncate,
+                False,
+            )
+            custom_embeddings_dict = dict(
+                zip(
+                    extracted_custom_content.loc[valid_custom_content_mask].index,
+                    custom_content_embeddings.get("embeddings", []),
+                )
+            )
+        else:
+            custom_embeddings_dict = {}
+
+        combined_df = combined_df.apply(
+            _add_custom_embeddings, embeddings=custom_embeddings_dict, result_target_field=result_target_field, axis=1
+        )
+
     return combined_df, {"trace_info": execution_trace_log}

{nv_ingest_api-2025.10.26.dev20251026 → nv_ingest_api-2025.10.28.dev20251028}/src/nv_ingest_api/util/dataloader/dataloader.py

@@ -254,22 +254,29 @@ else:
         file = None
         try:
             for file in paths:
+                if thread_stop.is_set():
+                    return
                 if isinstance(file, tuple):
                     video_file, audio_file = file
+                    if thread_stop.is_set():
+                        return
                     with open(video_file, "rb") as f:
                         video = f.read()
+                    if thread_stop.is_set():
+                        return
                     with open(audio_file, "rb") as f:
                         audio = f.read()
                     queue.put((video, audio))
                 else:
-                    if thread_stop:
+                    if thread_stop.is_set():
                         return
                     with open(file, "rb") as f:
                         queue.put(f.read())
         except Exception as e:
             logging.error(f"Error processing file {file}: {e}")
             queue.put(RuntimeError(f"Error processing file {file}: {e}"))
-        queue.put(StopIteration)
+        finally:
+            queue.put(StopIteration)
 
     class DataLoader:
         """
@@ -290,7 +297,7 @@ else:
         ):
             interface = interface if interface else MediaInterface()
             self.thread = None
-            self.thread_stop = False
+            self.thread_stop = threading.Event()
             self.queue = queue.Queue(size)
             self.path = Path(path)
             self.output_dir = output_dir
@@ -323,16 +330,20 @@ else:
             Reset itertor by stopping the thread and clearing the queue.
             """
             if self.thread:
-                self.thread_stop = True
+                self.thread_stop.set()
                 self.thread.join()
-            self.thread_stop = False
-            while self.queue.qsize() != 0:
-                with self.queue.mutex:
-                    self.queue.queue.clear()
+                self.thread = None
+            try:
+                while True:
+                    self.queue.get_nowait()
+            except Exception:
+                pass
+            finally:
+                self.thread_stop.clear()
 
         def __iter__(self):
             self.stop()
-            self.thread_stop = False
+            self.thread_stop.clear()
             self.thread = threading.Thread(
                 target=load_data,
                 args=(

{nv_ingest_api-2025.10.26.dev20251026 → nv_ingest_api-2025.10.28.dev20251028/src/nv_ingest_api.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-api
-Version: 2025.10.26.dev20251026
+Version: 2025.10.28.dev20251028
 Summary: Python module with core document ingestion functions.
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License
@@ -222,6 +222,7 @@ Requires-Dist: fsspec>=2025.5.1
 Requires-Dist: universal_pathlib>=0.2.6
 Requires-Dist: ffmpeg-python==0.2.0
 Requires-Dist: tritonclient
+Requires-Dist: glom
 Dynamic: license-file
 
 # nv-ingest-api

{nv_ingest_api-2025.10.26.dev20251026 → nv_ingest_api-2025.10.28.dev20251028}/src/nv_ingest_api.egg-info/SOURCES.txt

@@ -78,6 +78,7 @@ src/nv_ingest_api/internal/primitives/tracing/latency.py
 src/nv_ingest_api/internal/primitives/tracing/logging.py
 src/nv_ingest_api/internal/primitives/tracing/tagging.py
 src/nv_ingest_api/internal/schemas/__init__.py
+src/nv_ingest_api/internal/schemas/mixins.py
 src/nv_ingest_api/internal/schemas/extract/__init__.py
 src/nv_ingest_api/internal/schemas/extract/extract_audio_schema.py
 src/nv_ingest_api/internal/schemas/extract/extract_chart_schema.py

{nv_ingest_api-2025.10.26.dev20251026 → nv_ingest_api-2025.10.28.dev20251028}/src/nv_ingest_api.egg-info/requires.txt

@@ -7,3 +7,4 @@ fsspec>=2025.5.1
 universal_pathlib>=0.2.6
 ffmpeg-python==0.2.0
 tritonclient
+glom