nv-ingest 2025.11.19.dev20251119__py3-none-any.whl → 2025.12.5.dev20251205__py3-none-any.whl

nv_ingest/framework/orchestration/ray/stages/storage/image_storage.py

@@ -3,7 +3,9 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import logging
+import os
 from typing import Dict, Any, Optional
+from urllib.parse import urlparse
 
 import pandas as pd
 import ray
@@ -26,7 +28,8 @@ logger = logging.getLogger(__name__)
 @ray.remote
 class ImageStorageStage(RayActorStage):
     """
-    A Ray actor stage that stores images or structured content in MinIO and updates metadata with storage URLs.
+    A Ray actor stage that stores images or structured content using an fsspec-compatible backend and updates
+    metadata with storage URLs.
 
     This stage uses the validated configuration (ImageStorageModuleSchema) to process and store the DataFrame
     payload and updates the control message accordingly.
@@ -69,8 +72,16 @@ class ImageStorageStage(RayActorStage):
         task_config = remove_task_by_type(control_message, "store")
         # logger.debug("ImageStorageStage: Task configuration extracted: %s", pprint.pformat(task_config))
 
-        store_structured: bool = task_config.get("structured", True)
-        store_unstructured: bool = task_config.get("images", False)
+        stage_defaults = {
+            "structured": self.validated_config.structured,
+            "images": self.validated_config.images,
+            "storage_uri": self.validated_config.storage_uri,
+            "storage_options": self.validated_config.storage_options,
+            "public_base_url": self.validated_config.public_base_url,
+        }
+
+        store_structured: bool = task_config.get("structured", stage_defaults["structured"])
+        store_unstructured: bool = task_config.get("images", stage_defaults["images"])
 
         content_types: Dict[Any, Any] = {}
         if store_structured:
@@ -80,14 +91,34 @@ class ImageStorageStage(RayActorStage):
             content_types[ContentTypeEnum.IMAGE] = store_unstructured
 
         params: Dict[str, Any] = task_config.get("params", {})
-        params["content_types"] = content_types
 
-        logger.debug(f"Processing storage task with parameters: {params}")
+        storage_uri = task_config.get("storage_uri") or params.get("storage_uri") or stage_defaults["storage_uri"]
+        storage_options = {
+            **(stage_defaults["storage_options"] or {}),
+            **(task_config.get("storage_options") or {}),
+            **params.get("storage_options", {}),
+        }
+        if "public_base_url" in task_config:
+            public_base_url = task_config["public_base_url"]
+        else:
+            public_base_url = params.get("public_base_url", stage_defaults["public_base_url"])
+
+        storage_options = self._inject_storage_defaults(storage_uri, storage_options)
+
+        storage_params: Dict[str, Any] = {
+            "content_types": content_types,
+            "storage_uri": storage_uri,
+            "storage_options": storage_options,
+        }
+        if public_base_url:
+            storage_params["public_base_url"] = public_base_url
+
+        logger.debug("Processing storage task with parameters: %s", storage_params)
 
         # Store images or structured content.
         df_storage_ledger: pd.DataFrame = store_images_to_minio_internal(
             df_storage_ledger=df_payload,
-            task_config=params,
+            task_config=storage_params,
             storage_config={},
             execution_trace_log=None,
         )
@@ -98,3 +129,38 @@ class ImageStorageStage(RayActorStage):
         control_message.payload(df_storage_ledger)
 
         return control_message
+
+    @staticmethod
+    def _inject_storage_defaults(storage_uri: str, storage_options: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Populate storage options for common backends (e.g., MinIO/S3) using environment defaults.
+        """
+        parsed_scheme = urlparse(storage_uri).scheme.lower()
+        merged_options: Dict[str, Any] = {k: v for k, v in storage_options.items() if v is not None}
+
+        if parsed_scheme not in {"s3", "s3a", "s3n"}:
+            return merged_options
+
+        def _set_if_absent(key: str, env_var: str) -> None:
+            if key not in merged_options and env_var in os.environ:
+                merged_options[key] = os.environ[env_var]
+
+        _set_if_absent("key", "MINIO_ACCESS_KEY")
+        _set_if_absent("secret", "MINIO_SECRET_KEY")
+        if "token" not in merged_options and os.environ.get("MINIO_SESSION_TOKEN"):
+            merged_options["token"] = os.environ["MINIO_SESSION_TOKEN"]
+
+        client_kwargs = dict(merged_options.get("client_kwargs", {}))
+        endpoint = os.environ.get("MINIO_INTERNAL_ADDRESS")
+        if not endpoint:
+            endpoint = "http://minio:9000"
+        if endpoint and not endpoint.startswith(("http://", "https://")):
+            endpoint = f"http://{endpoint}"
+        client_kwargs.setdefault("endpoint_url", endpoint)
+        region = os.environ.get("MINIO_REGION")
+        if region:
+            client_kwargs.setdefault("region_name", region)
+        if client_kwargs:
+            merged_options["client_kwargs"] = client_kwargs
+
+        return merged_options

nv_ingest/pipeline/default_libmode_pipeline_impl.py

@@ -128,6 +128,13 @@ stages:
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|http
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
+      pdfium_config:
+        yolox_endpoints: [
+          $YOLOX_GRPC_ENDPOINT|"",
+          $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v2"
+        ]
+        yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|http
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -149,6 +156,13 @@ stages:
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|http
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
+      pdfium_config:
+        yolox_endpoints: [
+          $YOLOX_GRPC_ENDPOINT|"",
+          $YOLOX_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v2"
+        ]
+        yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|http
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -201,7 +215,7 @@ stages:
       endpoint_config:
         ocr_endpoints: [
           $OCR_GRPC_ENDPOINT|"",
-          $OCR_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/baidu/paddleocr"
+          $OCR_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr-v1"
         ]
         ocr_infer_protocol: $OCR_INFER_PROTOCOL|"http"
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
@@ -227,7 +241,7 @@ stages:
         yolox_infer_protocol: $YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL|"http"
         ocr_endpoints: [
           $OCR_GRPC_ENDPOINT|"",
-          $OCR_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/baidu/paddleocr"
+          $OCR_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr-v1"
         ]
         ocr_infer_protocol: $PADDLE_INFER_PROTOCOL|"http"
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
@@ -254,7 +268,7 @@ stages:
         yolox_infer_protocol: $YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL|"http"
         ocr_endpoints: [
           $OCR_GRPC_ENDPOINT|"",
-          $OCR_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/baidu/paddleocr"
+          $OCR_HTTP_ENDPOINT|"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr-v1"
         ]
         ocr_infer_protocol: $OCR_INFER_PROTOCOL|"http"
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY

nv_ingest/pipeline/default_pipeline_impl.py

@@ -123,7 +123,14 @@ stages:
       docx_extraction_config:
         yolox_endpoints: [
           $YOLOX_GRPC_ENDPOINT|"page-elements:8001",
-          $YOLOX_HTTP_ENDPOINT|"",
+          $YOLOX_HTTP_ENDPOINT|"http://page-elements:8000/v1/infer",
+        ]
+        yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|grpc
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
+      pdfium_config:
+        yolox_endpoints: [
+          $YOLOX_GRPC_ENDPOINT|"page-elements:8001",
+          $YOLOX_HTTP_ENDPOINT|"http://page-elements:8000/v1/infer",
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|grpc
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
@@ -148,6 +155,13 @@ stages:
         ]
         yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|grpc
         auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
+      pdfium_config:
+        yolox_endpoints: [
+          $YOLOX_GRPC_ENDPOINT|"page-elements:8001",
+          $YOLOX_HTTP_ENDPOINT|"http://page-elements:8000/v1/infer",
+        ]
+        yolox_infer_protocol: $YOLOX_INFER_PROTOCOL|grpc
+        auth_token: $NGC_API_KEY|$NVIDIA_API_KEY
     replicas:
       min_replicas: 0
       max_replicas:
@@ -372,6 +386,9 @@ stages:
     type: "stage"
     phase: 5  # RESPONSE
     actor: "nv_ingest.framework.orchestration.ray.stages.storage.image_storage:ImageStorageStage"
+    config:
+      storage_uri: $IMAGE_STORAGE_URI|"s3://nv-ingest/artifacts/store/images"
+      public_base_url: $IMAGE_STORAGE_PUBLIC_BASE_URL|""
     replicas:
       min_replicas: 0
       max_replicas:

{nv_ingest-2025.11.19.dev20251119.dist-info → nv_ingest-2025.12.5.dev20251205.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.11.19.dev20251119
+Version: 2025.12.5.dev20251205
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License
@@ -219,6 +219,8 @@ Requires-Dist: diskcache>=5.6.3
 Requires-Dist: fastapi>=0.115.6
 Requires-Dist: fastparquet>=2024.11.0
 Requires-Dist: fsspec>=2024.10.0
+Requires-Dist: universal_pathlib>=0.2.6
+Requires-Dist: s3fs>=2024.10.0
 Requires-Dist: gunicorn
 Requires-Dist: h11>=0.16.0
 Requires-Dist: httpx>=0.28.1

{nv_ingest-2025.11.19.dev20251119.dist-info → nv_ingest-2025.12.5.dev20251205.dist-info}/RECORD

@@ -64,7 +64,7 @@ nv_ingest/framework/orchestration/ray/stages/sinks/message_broker_task_sink.py,s
 nv_ingest/framework/orchestration/ray/stages/sources/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/ray/stages/sources/message_broker_task_source.py,sha256=LrqaWpWyuiAHlpXWKYSyHZJBFegGXfNlpCXrucbK5NM,24067
 nv_ingest/framework/orchestration/ray/stages/storage/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/framework/orchestration/ray/stages/storage/image_storage.py,sha256=WZN_-3Li-izDaPtk8IMrtn2os1ckT3U8Rb2PsfOWrcI,4009
+nv_ingest/framework/orchestration/ray/stages/storage/image_storage.py,sha256=f1iA7rjYFA1G1EXqFM6URUi_QRql1Y1OrnMPKONsSqo,6907
 nv_ingest/framework/orchestration/ray/stages/storage/store_embeddings.py,sha256=EUtwhSDf-qGLVEhWEInr1VaLsvpcHUSyzCmHQVai-Ps,3547
 nv_ingest/framework/orchestration/ray/stages/telemetry/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/ray/stages/telemetry/job_counter.py,sha256=jEtEUibqs6IS6QakrzWY9zmxSUzuBpg_hzXy2R-I10Y,2870
@@ -111,15 +111,15 @@ nv_ingest/framework/util/service/meta/ingest/ingest_service_meta.py,sha256=QS3uN
 nv_ingest/framework/util/telemetry/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/util/telemetry/global_stats.py,sha256=nq65pEEdiwjAfGiqsxG1CeQMC96O3CfQxsZuGFCY-ds,4554
 nv_ingest/pipeline/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/pipeline/default_libmode_pipeline_impl.py,sha256=yNJtjfHQyxtasGa1hQrvgX7UrPa7BAd0oog8EIN8Y_w,15592
-nv_ingest/pipeline/default_pipeline_impl.py,sha256=DhClC17lWUvtBIi2mCC4WkLWT0lxY-CFY0n6nriAxas,16017
+nv_ingest/pipeline/default_libmode_pipeline_impl.py,sha256=M31VN1xVTdoiNdjaSSPKEZr-yKhXDSwQ1hAVIkpJZLw,16232
+nv_ingest/pipeline/default_pipeline_impl.py,sha256=TW9N9UcgsBL5SG1pxuSdgBIyFpBORskbHCmvJBmIIuw,16770
 nv_ingest/pipeline/ingest_pipeline.py,sha256=wHAJhqAM2s8nbY-8itVogmSU-yVN4PZONGWcKnhzgfg,17794
 nv_ingest/pipeline/pipeline_schema.py,sha256=rLZZz2It2o2hVNWrZUJU8CarrqRei1fho3ZEMkkoBcg,17940
 nv_ingest/pipeline/config/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/pipeline/config/loaders.py,sha256=75Yr9WYO7j7ghvKTnYLfZXQZEH3J3VEZo5J4TunC_Us,7590
 nv_ingest/pipeline/config/replica_resolver.py,sha256=3zjh8gmepEYORFZRM4inq7GoBW0YL3gzUDiixUugjzQ,8899
-nv_ingest-2025.11.19.dev20251119.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-nv_ingest-2025.11.19.dev20251119.dist-info/METADATA,sha256=arJTf3Axy2qKAFDlP4lsKCftTw4vnJp3EECP6hmylYU,15092
-nv_ingest-2025.11.19.dev20251119.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nv_ingest-2025.11.19.dev20251119.dist-info/top_level.txt,sha256=sjb0ajIsgn3YgftSjZHlYO0HjYAIIhNuXG_AmywCvaU,10
-nv_ingest-2025.11.19.dev20251119.dist-info/RECORD,,
+nv_ingest-2025.12.5.dev20251205.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+nv_ingest-2025.12.5.dev20251205.dist-info/METADATA,sha256=xmWQacIeqshaQ9LxIv90A2xCEFrCGXzjYYIQBwKyHf0,15162
+nv_ingest-2025.12.5.dev20251205.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nv_ingest-2025.12.5.dev20251205.dist-info/top_level.txt,sha256=sjb0ajIsgn3YgftSjZHlYO0HjYAIIhNuXG_AmywCvaU,10
+nv_ingest-2025.12.5.dev20251205.dist-info/RECORD,,