nv-ingest 2025.11.22.dev20251122__tar.gz → 2026.1.6.dev20260106__tar.gz

{nv_ingest-2025.11.22.dev20251122 → nv_ingest-2026.1.6.dev20260106}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.11.22.dev20251122
+Version: 2026.1.6.dev20260106
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License
@@ -219,13 +219,15 @@ Requires-Dist: diskcache>=5.6.3
 Requires-Dist: fastapi>=0.115.6
 Requires-Dist: fastparquet>=2024.11.0
 Requires-Dist: fsspec>=2024.10.0
+Requires-Dist: universal_pathlib>=0.2.6
+Requires-Dist: s3fs>=2024.10.0
 Requires-Dist: gunicorn
 Requires-Dist: h11>=0.16.0
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: isodate>=0.7.2
 Requires-Dist: langdetect>=1.0.9
 Requires-Dist: minio>=7.2.12
-Requires-Dist: librosa>=0.10.2
+Requires-Dist: librosa==0.10.2
 Requires-Dist: opentelemetry-api>=1.27.0
 Requires-Dist: opentelemetry-exporter-otlp>=1.27.0
 Requires-Dist: opentelemetry-sdk>=1.27.0

{nv_ingest-2025.11.22.dev20251122 → nv_ingest-2026.1.6.dev20260106}/nv_ingest/api/main.py

@@ -23,7 +23,7 @@ logger = logging.getLogger(__name__)
 app = FastAPI(
     title="NV-Ingest Microservice",
     description="Service for ingesting heterogenous datatypes",
-    version="25.4.2",
+    version="26.1.0",
     contact={
         "name": "NVIDIA Corporation",
         "url": "https://nvidia.com",

{nv_ingest-2025.11.22.dev20251122 → nv_ingest-2026.1.6.dev20260106}/nv_ingest/api/v2/ingest.py

@@ -122,11 +122,16 @@ def get_pdf_split_page_count(client_override: Optional[int] = None) -> int:
         )
         return DEFAULT_PDF_SPLIT_PAGE_COUNT
 
-    if parsed <= 0:
-        logger.warning("PDF_SPLIT_PAGE_COUNT must be >= 1; received %s. Using 1.", parsed)
-        return 1
-
-    return parsed
+    clamped = max(MIN_PAGES, min(parsed, MAX_PAGES))
+    if clamped != parsed:
+        logger.warning(
+            "Env PDF_SPLIT_PAGE_COUNT=%s clamped to %s (min=%s, max=%s)",
+            parsed,
+            clamped,
+            MIN_PAGES,
+            MAX_PAGES,
+        )
+    return clamped
 
 
 def split_pdf_to_chunks(pdf_content: bytes, pages_per_chunk: int) -> List[Dict[str, Any]]:
@@ -955,7 +960,7 @@ async def submit_job_v2(
                         "subjob_order": subjob_ids,
                     }
                 )
-        elif document_types and payloads and document_types[0].lower() in ["mp4", "mov", "avi", "mp3", "wav"]:
+        elif document_types and payloads and document_types[0].lower() in ["mp4", "mov", "avi", "mp3", "wav", "mkv"]:
             document_type = document_types[0]
             upload_path = f"./{Path(original_source_id).name}"
             # dump the payload to a file, just came from client
@@ -1003,7 +1008,7 @@ async def submit_job_v2(
                         "page_count": chunk.get("page_count", 0),
                     }
                 )
-            logger.error(f"Removing uploaded file {upload_path}")
+            logger.debug(f"Removing uploaded file {upload_path}")
             os.remove(upload_path)
 
         if submission_items:

{nv_ingest-2025.11.22.dev20251122 → nv_ingest-2026.1.6.dev20260106}/nv_ingest/framework/orchestration/process/execution.py

@@ -162,6 +162,11 @@ def build_logging_config_from_env() -> LoggingConfig:
         if key not in os.environ:
             os.environ[key] = default_value
 
+    # For PRODUCTION mode, also suppress nv-ingest module INFO logs
+    if preset_level == "PRODUCTION":
+        logging.getLogger("nv_ingest").setLevel(logging.WARNING)
+        logging.getLogger("nv_ingest_api").setLevel(logging.WARNING)
+
     logger.info(f"Applied Ray logging preset: {preset_level}")
 
     # Get log level from environment, default to INFO
@@ -324,6 +329,7 @@ def launch_pipeline(
     pipeline_config = resolve_static_replicas(pipeline_config)
 
     # Pretty print the final pipeline configuration (after replica resolution)
+    # INFO level so it shows in docker/helm deployments; quiet mode suppresses in library mode
     pretty_output = pretty_print_pipeline_config(pipeline_config, config_path=None)
     logger.info("\n" + pretty_output)
 

{nv_ingest-2025.11.22.dev20251122 → nv_ingest-2026.1.6.dev20260106}/nv_ingest/framework/orchestration/ray/examples/pipeline_test_harness.py

@@ -150,7 +150,7 @@ if __name__ == "__main__":
     os.environ["OCR_GRPC_ENDPOINT"] = "localhost:8010"
     os.environ["OCR_INFER_PROTOCOL"] = "grpc"
     os.environ["OCR_MODEL_NAME"] = "paddle"
-    os.environ["NEMORETRIEVER_PARSE_HTTP_ENDPOINT"] = "https://integrate.api.nvidia.com/v1/chat/completions"
+    os.environ["NEMOTRON_PARSE_HTTP_ENDPOINT"] = "https://integrate.api.nvidia.com/v1/chat/completions"
     os.environ["VLM_CAPTION_ENDPOINT"] = "https://integrate.api.nvidia.com/v1/chat/completions"
     os.environ["VLM_CAPTION_MODEL_NAME"] = "nvidia/nemotron-nano-12b-v2-vl"
     logger.info("Environment variables set.")
@@ -170,23 +170,23 @@ if __name__ == "__main__":
         yolox_graphic_elements_auth,
         yolox_graphic_elements_protocol,
     ) = get_nim_service("yolox_graphic_elements")
-    nemoretriever_parse_grpc, nemoretriever_parse_http, nemoretriever_parse_auth, nemoretriever_parse_protocol = (
-        get_nim_service("nemoretriever_parse")
+    nemotron_parse_grpc, nemotron_parse_http, nemotron_parse_auth, nemotron_parse_protocol = get_nim_service(
+        "nemotron_parse"
     )
     ocr_grpc, ocr_http, ocr_auth, ocr_protocol = get_nim_service("ocr")
 
-    model_name = os.environ.get("NEMORETRIEVER_PARSE_MODEL_NAME", "nvidia/nemoretriever-parse")
+    model_name = os.environ.get("NEMOTRON_PARSE_MODEL_NAME", "nvidia/nemotron-parse")
     pdf_extractor_config = {
         "pdfium_config": {
             "auth_token": yolox_auth,  # All auth tokens are the same for the moment
             "yolox_endpoints": (yolox_grpc, yolox_http),
             "yolox_infer_protocol": yolox_protocol,
         },
-        "nemoretriever_parse_config": {
-            "auth_token": nemoretriever_parse_auth,
-            "nemoretriever_parse_endpoints": (nemoretriever_parse_grpc, nemoretriever_parse_http),
-            "nemoretriever_parse_infer_protocol": nemoretriever_parse_protocol,
-            "nemoretriever_parse_model_name": model_name,
+        "nemotron_parse_config": {
+            "auth_token": nemotron_parse_auth,
+            "nemotron_parse_endpoints": (nemotron_parse_grpc, nemotron_parse_http),
+            "nemotron_parse_infer_protocol": nemotron_parse_protocol,
+            "nemotron_parse_model_name": model_name,
             "yolox_endpoints": (yolox_grpc, yolox_http),
             "yolox_infer_protocol": yolox_protocol,
         },

nv_ingest-2026.1.6.dev20260106/nv_ingest/framework/orchestration/ray/stages/meta/udf_parallel_helper.py

@@ -0,0 +1,64 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Added this no-op UDF ray stage to the pipeline to help speed up the LLM api calls
+
+"""
+UDF Parallel Stage - A high-concurrency no-op stage for parallel UDF execution.
+
+This stage does nothing except pass messages through, but with high replica count
+it provides a parallel execution pool for UDFs to achieve N-way concurrency.
+"""
+
+import logging
+from typing import Any, Optional
+from pydantic import BaseModel
+import ray
+
+from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
+from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
+from nv_ingest_api.internal.primitives.tracing.tagging import traceable
+from nv_ingest_api.util.exception_handlers.decorators import (
+    nv_ingest_node_failure_try_except,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@ray.remote
+class UDFParallelStage(RayActorStage):
+    """
+    A no-op pass-through stage designed for parallel UDF execution.
+
+    This stage simply returns the input message unchanged, but when configured
+    with multiple replicas, it provides a high-concurrency pool for UDFs to
+    achieve parallel execution without blocking.
+    """
+
+    def __init__(self, config: BaseModel, stage_name: Optional[str] = None) -> None:
+        super().__init__(config, stage_name=stage_name)
+        logger.info(f"UDFParallelStage initialized: {stage_name}")
+
+    @nv_ingest_node_failure_try_except()
+    @traceable()
+    @udf_intercept_hook()
+    def on_data(self, message: Any) -> Any:
+        """
+        Pass-through processing that simply returns the message unchanged.
+
+        The @udf_intercept_hook decorator allows UDFs to target this stage,
+        and multiple replicas provide parallel execution capacity.
+
+        Parameters
+        ----------
+        message : Any
+            The incoming control message.
+
+        Returns
+        -------
+        Any
+            The unmodified control message.
+        """
+        # No-op: just return the message
+        return message

{nv_ingest-2025.11.22.dev20251122 → nv_ingest-2026.1.6.dev20260106}/nv_ingest/framework/orchestration/ray/stages/storage/image_storage.py

@@ -3,7 +3,9 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import logging
+import os
 from typing import Dict, Any, Optional
+from urllib.parse import urlparse
 
 import pandas as pd
 import ray
@@ -26,7 +28,8 @@ logger = logging.getLogger(__name__)
 @ray.remote
 class ImageStorageStage(RayActorStage):
     """
-    A Ray actor stage that stores images or structured content in MinIO and updates metadata with storage URLs.
+    A Ray actor stage that stores images or structured content using an fsspec-compatible backend and updates
+    metadata with storage URLs.
 
     This stage uses the validated configuration (ImageStorageModuleSchema) to process and store the DataFrame
     payload and updates the control message accordingly.
@@ -69,8 +72,16 @@ class ImageStorageStage(RayActorStage):
         task_config = remove_task_by_type(control_message, "store")
         # logger.debug("ImageStorageStage: Task configuration extracted: %s", pprint.pformat(task_config))
 
-        store_structured: bool = task_config.get("structured", True)
-        store_unstructured: bool = task_config.get("images", False)
+        stage_defaults = {
+            "structured": self.validated_config.structured,
+            "images": self.validated_config.images,
+            "storage_uri": self.validated_config.storage_uri,
+            "storage_options": self.validated_config.storage_options,
+            "public_base_url": self.validated_config.public_base_url,
+        }
+
+        store_structured: bool = task_config.get("structured", stage_defaults["structured"])
+        store_unstructured: bool = task_config.get("images", stage_defaults["images"])
 
         content_types: Dict[Any, Any] = {}
         if store_structured:
@@ -80,14 +91,34 @@ class ImageStorageStage(RayActorStage):
             content_types[ContentTypeEnum.IMAGE] = store_unstructured
 
         params: Dict[str, Any] = task_config.get("params", {})
-        params["content_types"] = content_types
 
-        logger.debug(f"Processing storage task with parameters: {params}")
+        storage_uri = task_config.get("storage_uri") or params.get("storage_uri") or stage_defaults["storage_uri"]
+        storage_options = {
+            **(stage_defaults["storage_options"] or {}),
+            **(task_config.get("storage_options") or {}),
+            **params.get("storage_options", {}),
+        }
+        if "public_base_url" in task_config:
+            public_base_url = task_config["public_base_url"]
+        else:
+            public_base_url = params.get("public_base_url", stage_defaults["public_base_url"])
+
+        storage_options = self._inject_storage_defaults(storage_uri, storage_options)
+
+        storage_params: Dict[str, Any] = {
+            "content_types": content_types,
+            "storage_uri": storage_uri,
+            "storage_options": storage_options,
+        }
+        if public_base_url:
+            storage_params["public_base_url"] = public_base_url
+
+        logger.debug("Processing storage task with parameters: %s", storage_params)
 
         # Store images or structured content.
         df_storage_ledger: pd.DataFrame = store_images_to_minio_internal(
             df_storage_ledger=df_payload,
-            task_config=params,
+            task_config=storage_params,
             storage_config={},
             execution_trace_log=None,
         )
@@ -98,3 +129,38 @@ class ImageStorageStage(RayActorStage):
         control_message.payload(df_storage_ledger)
 
         return control_message
+
+    @staticmethod
+    def _inject_storage_defaults(storage_uri: str, storage_options: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Populate storage options for common backends (e.g., MinIO/S3) using environment defaults.
+        """
+        parsed_scheme = urlparse(storage_uri).scheme.lower()
+        merged_options: Dict[str, Any] = {k: v for k, v in storage_options.items() if v is not None}
+
+        if parsed_scheme not in {"s3", "s3a", "s3n"}:
+            return merged_options
+
+        def _set_if_absent(key: str, env_var: str) -> None:
+            if key not in merged_options and env_var in os.environ:
+                merged_options[key] = os.environ[env_var]
+
+        _set_if_absent("key", "MINIO_ACCESS_KEY")
+        _set_if_absent("secret", "MINIO_SECRET_KEY")
+        if "token" not in merged_options and os.environ.get("MINIO_SESSION_TOKEN"):
+            merged_options["token"] = os.environ["MINIO_SESSION_TOKEN"]
+
+        client_kwargs = dict(merged_options.get("client_kwargs", {}))
+        endpoint = os.environ.get("MINIO_INTERNAL_ADDRESS")
+        if not endpoint:
+            endpoint = "http://minio:9000"
+        if endpoint and not endpoint.startswith(("http://", "https://")):
+            endpoint = f"http://{endpoint}"
+        client_kwargs.setdefault("endpoint_url", endpoint)
+        region = os.environ.get("MINIO_REGION")
+        if region:
+            client_kwargs.setdefault("region_name", region)
+        if client_kwargs:
+            merged_options["client_kwargs"] = client_kwargs
+
+        return merged_options

{nv_ingest-2025.11.22.dev20251122 → nv_ingest-2026.1.6.dev20260106}/nv_ingest/framework/orchestration/ray/util/pipeline/pipeline_runners.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import logging
+import os
 from typing import Union, Optional, TextIO
 
 
@@ -23,6 +24,34 @@ from nv_ingest.framework.orchestration.execution.helpers import (
 logger = logging.getLogger(__name__)
 
 
+def _configure_quiet_mode():
+    """
+    Configure environment for quiet/production logging in library mode.
+
+    Sets INGEST_RAY_LOG_LEVEL=PRODUCTION if not already set by user, which:
+    - Sets Ray logging to ERROR level (suppresses INFO/WARNING)
+    - Disables Ray usage stats collection
+    - Disables Ray import warnings
+
+    Also silences other common warnings that are noisy in library mode.
+    """
+    # Only set if user hasn't explicitly configured
+    if "INGEST_RAY_LOG_LEVEL" not in os.environ:
+        os.environ["INGEST_RAY_LOG_LEVEL"] = "PRODUCTION"
+
+    # Silence Ray accelerator env var warning
+    if "RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO" not in os.environ:
+        os.environ["RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO"] = "0"
+
+    # Disable OTEL tracing export errors (no collector expected in library mode)
+    if "OTEL_SDK_DISABLED" not in os.environ:
+        os.environ["OTEL_SDK_DISABLED"] = "true"
+
+    # Set nv-ingest module loggers to WARNING to suppress INFO level startup messages
+    logging.getLogger("nv_ingest").setLevel(logging.WARNING)
+    logging.getLogger("nv_ingest_api").setLevel(logging.WARNING)
+
+
 def run_pipeline(
     pipeline_config: Optional[PipelineConfigSchema] = None,
     block: bool = True,
@@ -32,6 +61,7 @@ def run_pipeline(
     stdout: Optional[TextIO] = None,
     stderr: Optional[TextIO] = None,
     libmode: bool = True,
+    quiet: Optional[bool] = None,
 ) -> Union[RayPipelineInterface, float, RayPipelineSubprocessInterface]:
     """
     Launch and manage a pipeline using configuration.
@@ -65,6 +95,10 @@ def run_pipeline(
     libmode : bool, default=True
         If True and pipeline_config is None, loads the default libmode pipeline configuration.
         If False, requires pipeline_config to be provided.
+    quiet : Optional[bool], default=None
+        If True, configures logging for minimal output (PRODUCTION preset, suppresses
+        INFO-level startup messages). If None, defaults to True when libmode=True.
+        Set to False to see verbose startup logs even in library mode.
 
     Returns
     -------
@@ -83,6 +117,12 @@ def run_pipeline(
     Exception
         Any other exceptions raised during pipeline launch or configuration.
     """
+    # Configure quiet mode for library mode by default (unless explicitly disabled)
+    if quiet is None:
+        quiet = libmode
+    if quiet:
+        _configure_quiet_mode()
+
     # Resolve configuration
     config = resolve_pipeline_config(pipeline_config, libmode)
     overrides = create_runtime_overrides(disable_dynamic_scaling, dynamic_memory_threshold)

{nv_ingest-2025.11.22.dev20251122 → nv_ingest-2026.1.6.dev20260106}/nv_ingest/pipeline/config/replica_resolver.py

@@ -11,6 +11,7 @@ consumption stays within the static_memory_threshold.
 """
 
 import logging
+import os
 from typing import List
 from copy import deepcopy
 
@@ -102,8 +103,17 @@ def resolve_static_replicas(pipeline_config: PipelineConfigSchema) -> PipelineCo
 
     logger.info(f"Total baseline memory demand: {total_memory_demand_mb}MB from {len(non_static_stages)} stages")
 
-    # Check if we need to scale down
-    if total_memory_demand_mb <= available_memory_mb:
+    # Optional bypass of global memory-based scale down via environment variable
+    bypass_env = os.getenv("NV_INGEST_BYPASS_STATIC_MEMORY_SCALE_DOWN", "").strip().lower()
+    bypass_scale_down = bypass_env in ("1", "true", "yes", "on")
+
+    # Check if we need to scale down (unless bypassed)
+    if bypass_scale_down:
+        logger.warning(
+            "Bypassing static memory-based replica scale-down due to NV_INGEST_BYPASS_STATIC_MEMORY_SCALE_DOWN"
+        )
+        scaling_factor = 1.0
+    elif total_memory_demand_mb <= available_memory_mb:
         logger.info("Memory demand within threshold, applying baseline replica counts")
         scaling_factor = 1.0
     else:

{nv_ingest-2025.11.22.dev20251122 → nv_ingest-2026.1.6.dev20260106}/nv_ingest/pipeline/default_libmode_pipeline_impl.py

@@ -68,20 +68,20 @@ stages:
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
         yolox_endpoints: [
           $YOLOX_GRPC_ENDPOINT|"",
-          $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v2"
+          $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3"
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|http
-      nemoretriever_parse_config:
+      nemotron_parse_config:
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
-        nemoretriever_parse_endpoints: [
-          $NEMORETRIEVER_PARSE_GRPC_ENDPOINT|"",
-          $NEMORETRIEVER_PARSE_HTTP_ENDPOINT|"https://integrate.api.nvidia.com/v1/chat/completions"
+        nemotron_parse_endpoints: [
+          $NEMOTRON_PARSE_GRPC_ENDPOINT|"",
+          $NEMOTRON_PARSE_HTTP_ENDPOINT|"https://integrate.api.nvidia.com/v1/chat/completions"
         ]
-        nemoretriever_parse_infer_protocol: $NEMORETRIEVER_PARSE_INFER_PROTOCOL|http
-        nemoretriever_parse_model_name: $NEMORETRIEVER_PARSE_MODEL_NAME|"nvidia/nemoretriever-parse"
+        nemotron_parse_infer_protocol: $NEMOTRON_PARSE_INFER_PROTOCOL|http
+        nemotron_parse_model_name: $NEMOTRON_PARSE_MODEL_NAME|"nvidia/nemotron-parse"
         yolox_endpoints: [
           $YOLOX_GRPC_ENDPOINT|"",
-          $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v2"
+          $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3"
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|http
     replicas:
@@ -124,7 +124,14 @@ stages:
       docx_extraction_config:
         yolox_endpoints: [
           $YOLOX_GRPC_ENDPOINT|"",
-          $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v2"
+          $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3"
+        ]
+        yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|http
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
+      pdfium_config:
+        yolox_endpoints: [
+          $YOLOX_GRPC_ENDPOINT|"",
+          $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3"
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|http
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
@@ -145,7 +152,14 @@ stages:
       pptx_extraction_config:
         yolox_endpoints: [
           $YOLOX_GRPC_ENDPOINT|"",
-          $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v2"
+          $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3"
+        ]
+        yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|http
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
+      pdfium_config:
+        yolox_endpoints: [
+          $YOLOX_GRPC_ENDPOINT|"",
+          $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3"
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|http
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
@@ -166,7 +180,7 @@ stages:
       image_extraction_config:
         yolox_endpoints: [
           $YOLOX_GRPC_ENDPOINT|"",
-          $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v2"
+          $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3"
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|http
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
@@ -201,7 +215,7 @@ stages:
       endpoint_config:
         ocr_endpoints: [
           $OCR_GRPC_ENDPOINT|"",
-          $OCR_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/baidu/paddleocr"
+          $OCR_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr-v1"
         ]
         ocr_infer_protocol: $OCR_INFER_PROTOCOL|"http"
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
@@ -227,9 +241,9 @@ stages:
         yolox_infer_protocol: $YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL|"http"
         ocr_endpoints: [
           $OCR_GRPC_ENDPOINT|"",
-          $OCR_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/baidu/paddleocr"
+          $OCR_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr-v1"
         ]
-        ocr_infer_protocol: $PADDLE_INFER_PROTOCOL|"http"
+        ocr_infer_protocol: $OCR_INFER_PROTOCOL|"http"
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
@@ -254,7 +268,7 @@ stages:
         yolox_infer_protocol: $YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL|"http"
         ocr_endpoints: [
           $OCR_GRPC_ENDPOINT|"",
-          $OCR_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/baidu/paddleocr"
+          $OCR_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr-v1"
         ]
         ocr_infer_protocol: $OCR_INFER_PROTOCOL|"http"
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
@@ -318,9 +332,10 @@ stages:
     actor: "nv_ingest.framework.orchestration.ray.stages.transforms.image_caption:ImageCaptionTransformStage"
     config:
       api_key: $NGC_API_KEY|$NVIDIA_API_KEY
-      endpoint_url: $VLM_CAPTION_ENDPOINT|"http://vlm:8000/v1/chat/completions"
+      endpoint_url: $VLM_CAPTION_ENDPOINT|"https://integrate.api.nvidia.com/v1/chat/completions"
       model_name: $VLM_CAPTION_MODEL_NAME|"nvidia/nemotron-nano-12b-v2-vl"
-      prompt: "Caption the content of this image:"
+      prompt: $VLM_CAPTION_PROMPT|"Caption the content of this image:"
+      system_prompt: $VLM_CAPTION_SYSTEM_PROMPT|"/no_think"
     replicas:
       min_replicas: 0
       max_replicas:

{nv_ingest-2025.11.22.dev20251122 → nv_ingest-2026.1.6.dev20260106}/nv_ingest/pipeline/default_pipeline_impl.py

@@ -70,14 +70,14 @@ stages:
           $YOLOX_HTTP_ENDPOINT|"http://page-elements:8000/v1/infer",
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|grpc
-      nemoretriever_parse_config:
+      nemotron_parse_config:
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
-        nemoretriever_parse_endpoints: [
-          $NEMORETRIEVER_PARSE_GRPC_ENDPOINT|"",
-          $NEMORETRIEVER_PARSE_HTTP_ENDPOINT|"http://nemoretriever-parse:8000/v1/chat/completions",
+        nemotron_parse_endpoints: [
+          $NEMOTRON_PARSE_GRPC_ENDPOINT|"",
+          $NEMOTRON_PARSE_HTTP_ENDPOINT|"http://nemotron-parse:8000/v1/chat/completions",
         ]
-        nemoretriever_parse_infer_protocol: $NEMORETRIEVER_PARSE_INFER_PROTOCOL|http
-        nemoretriever_parse_model_name: $NEMORETRIEVER_PARSE_MODEL_NAME|"nvidia/nemoretriever-parse"
+        nemotron_parse_infer_protocol: $NEMOTRON_PARSE_INFER_PROTOCOL|http
+        nemotron_parse_model_name: $NEMOTRON_PARSE_MODEL_NAME|"nvidia/nemotron-parse"
         yolox_endpoints: [
           $YOLOX_GRPC_ENDPOINT|"page-elements:8001",
           $YOLOX_HTTP_ENDPOINT|"http://page-elements:8000/v1/infer",
@@ -123,7 +123,14 @@ stages:
       docx_extraction_config:
         yolox_endpoints: [
           $YOLOX_GRPC_ENDPOINT|"page-elements:8001",
-          $YOLOX_HTTP_ENDPOINT|"",
+          $YOLOX_HTTP_ENDPOINT|"http://page-elements:8000/v1/infer",
+        ]
+        yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|grpc
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
+      pdfium_config:
+        yolox_endpoints: [
+          $YOLOX_GRPC_ENDPOINT|"page-elements:8001",
+          $YOLOX_HTTP_ENDPOINT|"http://page-elements:8000/v1/infer",
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|grpc
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
@@ -148,6 +155,13 @@ stages:
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|grpc
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
+      pdfium_config:
+        yolox_endpoints: [
+          $YOLOX_GRPC_ENDPOINT|"page-elements:8001",
+          $YOLOX_HTTP_ENDPOINT|"http://page-elements:8000/v1/infer",
+        ]
+        yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|grpc
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -340,7 +354,8 @@ stages:
       api_key: $NGC_API_KEY|$NVIDIA_API_KEY
       model_name: $VLM_CAPTION_MODEL_NAME|"nvidia/nemotron-nano-12b-v2-vl"
       endpoint_url: $VLM_CAPTION_ENDPOINT|"http://vlm:8000/v1/chat/completions"
-      prompt: "Caption the content of this image:"
+      prompt: $VLM_CAPTION_PROMPT|"Caption the content of this image:"
+      system_prompt: $VLM_CAPTION_SYSTEM_PROMPT|"/no_think"
     replicas:
       min_replicas: 0
       max_replicas:
@@ -372,6 +387,9 @@ stages:
     type: "stage"
     phase: 5  # RESPONSE
     actor: "nv_ingest.framework.orchestration.ray.stages.storage.image_storage:ImageStorageStage"
+    config:
+      storage_uri: $IMAGE_STORAGE_URI|"s3://nv-ingest/artifacts/store/images"
+      public_base_url: $IMAGE_STORAGE_PUBLIC_BASE_URL|""
     replicas:
       min_replicas: 0
       max_replicas:

{nv_ingest-2025.11.22.dev20251122 → nv_ingest-2026.1.6.dev20260106}/nv_ingest.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.11.22.dev20251122
+Version: 2026.1.6.dev20260106
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License
@@ -219,13 +219,15 @@ Requires-Dist: diskcache>=5.6.3
 Requires-Dist: fastapi>=0.115.6
 Requires-Dist: fastparquet>=2024.11.0
 Requires-Dist: fsspec>=2024.10.0
+Requires-Dist: universal_pathlib>=0.2.6
+Requires-Dist: s3fs>=2024.10.0
 Requires-Dist: gunicorn
 Requires-Dist: h11>=0.16.0
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: isodate>=0.7.2
 Requires-Dist: langdetect>=1.0.9
 Requires-Dist: minio>=7.2.12
-Requires-Dist: librosa>=0.10.2
+Requires-Dist: librosa==0.10.2
 Requires-Dist: opentelemetry-api>=1.27.0
 Requires-Dist: opentelemetry-exporter-otlp>=1.27.0
 Requires-Dist: opentelemetry-sdk>=1.27.0

{nv_ingest-2025.11.22.dev20251122 → nv_ingest-2026.1.6.dev20260106}/nv_ingest.egg-info/SOURCES.txt

@@ -63,6 +63,7 @@ nv_ingest/framework/orchestration/ray/stages/meta/ray_actor_edge_base.py
 nv_ingest/framework/orchestration/ray/stages/meta/ray_actor_sink_stage_base.py
 nv_ingest/framework/orchestration/ray/stages/meta/ray_actor_source_stage_base.py
 nv_ingest/framework/orchestration/ray/stages/meta/ray_actor_stage_base.py
+nv_ingest/framework/orchestration/ray/stages/meta/udf_parallel_helper.py
 nv_ingest/framework/orchestration/ray/stages/mutate/__init__.py
 nv_ingest/framework/orchestration/ray/stages/mutate/image_dedup.py
 nv_ingest/framework/orchestration/ray/stages/mutate/image_filter.py

{nv_ingest-2025.11.22.dev20251122 → nv_ingest-2026.1.6.dev20260106}/nv_ingest.egg-info/requires.txt

@@ -4,13 +4,15 @@ diskcache>=5.6.3
 fastapi>=0.115.6
 fastparquet>=2024.11.0
 fsspec>=2024.10.0
+universal_pathlib>=0.2.6
+s3fs>=2024.10.0
 gunicorn
 h11>=0.16.0
 httpx>=0.28.1
 isodate>=0.7.2
 langdetect>=1.0.9
 minio>=7.2.12
-librosa>=0.10.2
+librosa==0.10.2
 opentelemetry-api>=1.27.0
 opentelemetry-exporter-otlp>=1.27.0
 opentelemetry-sdk>=1.27.0

{nv_ingest-2025.11.22.dev20251122 → nv_ingest-2026.1.6.dev20260106}/pyproject.toml

@@ -26,13 +26,15 @@ dependencies = [
     "fastapi>=0.115.6",
     "fastparquet>=2024.11.0",
     "fsspec>=2024.10.0",
+    "universal_pathlib>=0.2.6",
+    "s3fs>=2024.10.0",
     "gunicorn",
     "h11>=0.16.0", # Must pin at or above 0.16.0 for CVE mitigation
     "httpx>=0.28.1",
     "isodate>=0.7.2",
     "langdetect>=1.0.9",
     "minio>=7.2.12",
-    "librosa>=0.10.2",
+    "librosa==0.10.2",
     "opentelemetry-api>=1.27.0",
     "opentelemetry-exporter-otlp>=1.27.0",
     "opentelemetry-sdk>=1.27.0",