nv-ingest 2025.10.14.dev20251014__tar.gz → 2025.10.16.dev20251016__tar.gz

{nv_ingest-2025.10.14.dev20251014 → nv_ingest-2025.10.16.dev20251016}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.10.14.dev20251014
+Version: 2025.10.16.dev20251016
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest-2025.10.14.dev20251014 → nv_ingest-2025.10.16.dev20251016}/nv_ingest/api/v2/ingest.py

@@ -45,9 +45,30 @@ router = APIRouter()
 DEFAULT_PDF_SPLIT_PAGE_COUNT = 32
 
 
-def get_pdf_split_page_count() -> int:
-    """Resolve the configured page chunk size for PDF splitting."""
+def get_pdf_split_page_count(client_override: Optional[int] = None) -> int:
+    """
+    Resolve the page chunk size for PDF splitting with client override support.
 
+    Priority: client_override (clamped) > env var > default (32)
+    Enforces boundaries: min=1, max=128
+    """
+    MIN_PAGES = 1
+    MAX_PAGES = 128
+
+    # Client override takes precedence if provided
+    if client_override is not None:
+        clamped = max(MIN_PAGES, min(client_override, MAX_PAGES))
+        if clamped != client_override:
+            logger.warning(
+                "Client requested split_page_count=%s; clamped to %s (min=%s, max=%s)",
+                client_override,
+                clamped,
+                MIN_PAGES,
+                MAX_PAGES,
+            )
+        return clamped
+
+    # Fall back to environment variable
     raw_value = os.environ.get("PDF_SPLIT_PAGE_COUNT")
     if raw_value is None:
         return DEFAULT_PDF_SPLIT_PAGE_COUNT
@@ -530,6 +551,10 @@ async def submit_job_v2(
         # Parse job spec
         job_spec_dict = json.loads(job_spec.payload)
 
+        # Extract PDF configuration if provided by client
+        pdf_config = job_spec_dict.get("pdf_config", {})
+        client_split_page_count = pdf_config.get("split_page_count") if pdf_config else None
+
         # Extract document type and payload from the proper structure
         job_payload = job_spec_dict.get("job_payload", {})
         document_types = job_payload.get("document_type", [])
@@ -546,12 +571,12 @@ async def submit_job_v2(
             # Decode the payload to check page count
             pdf_content = base64.b64decode(payloads[0])
             page_count = get_pdf_page_count(pdf_content)
-            pages_per_chunk = get_pdf_split_page_count()
+            pages_per_chunk = get_pdf_split_page_count(client_override=client_split_page_count)
 
             # Split if the document has more pages than our chunk size
             if page_count > pages_per_chunk:
                 logger.warning(
-                    "[dev-reload-check] Splitting PDF %s into %s-page chunks (total pages: %s)",
+                    "Splitting PDF %s into %s-page chunks (total pages: %s)",
                     original_source_name,
                     pages_per_chunk,
                     page_count,
@@ -599,6 +624,7 @@ async def submit_job_v2(
 
                 parent_metadata: Dict[str, Any] = {
                     "total_pages": page_count,
+                    "pages_per_chunk": pages_per_chunk,
                     "original_source_id": original_source_id,
                     "original_source_name": original_source_name,
                     "document_type": document_types[0] if document_types else "pdf",

{nv_ingest-2025.10.14.dev20251014 → nv_ingest-2025.10.16.dev20251016}/nv_ingest/pipeline/default_libmode_pipeline_impl.py

@@ -65,14 +65,14 @@ stages:
     actor: "nv_ingest.framework.orchestration.ray.stages.extractors.pdf_extractor:PDFExtractorStage"
     config:
       pdfium_config:
-        auth_token: $NGC_API_KEY|""
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
         yolox_endpoints: [
           $YOLOX_GRPC_ENDPOINT|"",
           $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v2"
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|http
       nemoretriever_parse_config:
-        auth_token: $NGC_API_KEY|""
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
         nemoretriever_parse_endpoints: [
           $NEMORETRIEVER_PARSE_GRPC_ENDPOINT|"",
           $NEMORETRIEVER_PARSE_HTTP_ENDPOINT|"https://integrate.api.nvidia.com/v1/chat/completions"
@@ -106,7 +106,7 @@ stages:
         ]
         function_id: $AUDIO_FUNCTION_ID|"1598d209-5e27-4d3c-8079-4751568b1081"
         audio_infer_protocol: $AUDIO_INFER_PROTOCOL|grpc
-        auth_token: $NGC_API_KEY|""
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -127,7 +127,7 @@ stages:
           $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v2"
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|http
-        auth_token: $NGC_API_KEY|""
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -148,7 +148,7 @@ stages:
           $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v2"
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|http
-        auth_token: $NGC_API_KEY|""
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -169,7 +169,7 @@ stages:
           $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v2"
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|http
-        auth_token: $NGC_API_KEY|""
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -200,11 +200,11 @@ stages:
     config:
       endpoint_config:
         ocr_endpoints: [
-          $OCR_GRPC_ENDPOINT|"grpc.nvcf.nvidia.com:443",
-          $OCR_HTTP_ENDPOINT|""
+          $OCR_GRPC_ENDPOINT|"",
+          $OCR_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/baidu/paddleocr"
         ]
-        ocr_infer_protocol: $OCR_INFER_PROTOCOL|grpc
-        auth_token: $NGC_API_KEY|""
+        ocr_infer_protocol: $OCR_INFER_PROTOCOL|"http"
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -230,7 +230,7 @@ stages:
           $OCR_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/baidu/paddleocr"
         ]
         ocr_infer_protocol: $PADDLE_INFER_PROTOCOL|"http"
-        auth_token: $NGC_API_KEY|""
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -257,7 +257,7 @@ stages:
           $OCR_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/baidu/paddleocr"
         ]
         ocr_infer_protocol: $OCR_INFER_PROTOCOL|"http"
-        auth_token: $NGC_API_KEY|""
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -317,7 +317,7 @@ stages:
     phase: 4  # TRANSFORM
     actor: "nv_ingest.framework.orchestration.ray.stages.transforms.image_caption:ImageCaptionTransformStage"
     config:
-      api_key: $NGC_API_KEY|""
+      api_key: $NGC_API_KEY|$NVIDIA_API_KEY
       endpoint_url: $VLM_CAPTION_ENDPOINT|"https://integrate.api.nvidia.com/v1/chat/completions"
       model_name: $VLM_CAPTION_MODEL_NAME|"nvidia/llama-3.1-nemotron-nano-vl-8b-v1"
       prompt: "Caption the content of this image:"
@@ -335,7 +335,7 @@ stages:
     phase: 4  # TRANSFORM
     actor: "nv_ingest.framework.orchestration.ray.stages.transforms.text_embed:TextEmbeddingTransformStage"
     config:
-      api_key: $NGC_API_KEY|""
+      api_key: $NGC_API_KEY|$NVIDIA_API_KEY
       embedding_model: $EMBEDDING_NIM_MODEL_NAME|"nvidia/llama-3.2-nv-embedqa-1b-v2"
       embedding_nim_endpoint: $EMBEDDING_NIM_ENDPOINT|"https://integrate.api.nvidia.com/v1"
     replicas:

{nv_ingest-2025.10.14.dev20251014 → nv_ingest-2025.10.16.dev20251016}/nv_ingest/pipeline/default_pipeline_impl.py

@@ -64,14 +64,14 @@ stages:
     actor: "nv_ingest.framework.orchestration.ray.stages.extractors.pdf_extractor:PDFExtractorStage"
     config:
       pdfium_config:
-        auth_token: $NGC_API_KEY|""
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
         yolox_endpoints: [
           $YOLOX_GRPC_ENDPOINT|"page-elements:8001",
           $YOLOX_HTTP_ENDPOINT|"http://page-elements:8000/v1/infer",
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|grpc
       nemoretriever_parse_config:
-        auth_token: $NGC_API_KEY|""
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
         nemoretriever_parse_endpoints: [
           $NEMORETRIEVER_PARSE_GRPC_ENDPOINT|"",
           $NEMORETRIEVER_PARSE_HTTP_ENDPOINT|"http://nemoretriever-parse:8000/v1/chat/completions",
@@ -105,7 +105,7 @@ stages:
         ]
         function_id: $AUDIO_FUNCTION_ID|""
         audio_infer_protocol: $AUDIO_INFER_PROTOCOL|grpc
-        auth_token: $NGC_API_KEY|""
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -126,7 +126,7 @@ stages:
           $YOLOX_HTTP_ENDPOINT|"",
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|grpc
-        auth_token: $NGC_API_KEY|""
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -147,7 +147,7 @@ stages:
           $YOLOX_HTTP_ENDPOINT|"http://page-elements:8000/v1/infer",
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|grpc
-        auth_token: $NGC_API_KEY|""
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -168,7 +168,7 @@ stages:
           $YOLOX_HTTP_ENDPOINT|"http://page-elements:8000/v1/infer",
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|grpc
-        auth_token: $NGC_API_KEY|""
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -203,7 +203,7 @@ stages:
           $OCR_HTTP_ENDPOINT|"http://ocr:8000/v1/infer",
         ]
         ocr_infer_protocol: $OCR_INFER_PROTOCOL|grpc
-        auth_token: $NGC_API_KEY|""
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -229,7 +229,7 @@ stages:
           $OCR_HTTP_ENDPOINT|"http://ocr:8000/v1/infer",
         ]
         ocr_infer_protocol: $OCR_INFER_PROTOCOL|grpc
-        auth_token: $NGC_API_KEY|""
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -256,7 +256,7 @@ stages:
           $OCR_HTTP_ENDPOINT|""
         ]
         ocr_infer_protocol: $OCR_INFER_PROTOCOL|grpc
-        auth_token: $NGC_API_KEY|""
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -316,7 +316,7 @@ stages:
     phase: 4  # TRANSFORM
     actor: "nv_ingest.framework.orchestration.ray.stages.transforms.image_caption:ImageCaptionTransformStage"
     config:
-      api_key: $NGC_API_KEY|""
+      api_key: $NGC_API_KEY|$NVIDIA_API_KEY
       model_name: $VLM_CAPTION_MODEL_NAME|"nvidia/llama-3.1-nemotron-nano-vl-8b-v1"
       prompt: "Caption the content of this image:"
     replicas:
@@ -333,7 +333,7 @@ stages:
     phase: 4  # TRANSFORM
     actor: "nv_ingest.framework.orchestration.ray.stages.transforms.text_embed:TextEmbeddingTransformStage"
     config:
-      api_key: $NGC_API_KEY|""
+      api_key: $NGC_API_KEY|$NVIDIA_API_KEY
       embedding_model: $EMBEDDING_NIM_MODEL_NAME|"nvidia/llama-3.2-nv-embedqa-1b-v2"
       embedding_nim_endpoint: $EMBEDDING_NIM_ENDPOINT|"http://embedding:8000/v1"
     replicas:

{nv_ingest-2025.10.14.dev20251014 → nv_ingest-2025.10.16.dev20251016}/nv_ingest.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.10.14.dev20251014
+Version: 2025.10.16.dev20251016
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License