nv-ingest 2025.6.2.dev20250602__py3-none-any.whl → 2025.6.24.dev20250625__py3-none-any.whl

nv_ingest/framework/orchestration/ray/examples/pipeline_test_harness.py

@@ -155,7 +155,7 @@ if __name__ == "__main__":
     logger.info("Environment variables set.")
 
     image_caption_endpoint_url = "https://integrate.api.nvidia.com/v1/chat/completions"
-    image_caption_model_name = "meta/llama-3.2-11b-vision-instruct"
+    model_name = "meta/llama-3.2-11b-vision-instruct"
     yolox_grpc, yolox_http, yolox_auth, yolox_protocol = get_nim_service("yolox")
     (
         yolox_table_structure_grpc,
@@ -228,7 +228,7 @@ if __name__ == "__main__":
     image_caption_config = {
         "api_key": yolox_auth,
         "endpoint_url": image_caption_endpoint_url,
-        "image_caption_model_name": image_caption_model_name,
+        "model_name": model_name,
         "prompt": "Caption the content of this image:",
     }
     logger.info("Service configuration retrieved from get_nim_service and environment variables.")

nv_ingest/framework/orchestration/ray/primitives/pipeline_topology.py

@@ -555,7 +555,7 @@ class PipelineTopology:
             return None
 
     def get_connections(self) -> Dict[str, List[Tuple[str, int]]]:
-        """Returns a shallow copy of the connections dictionary."""
+        """Returns a shallow copy of the connection dictionary."""
         with self._lock:
             # Shallow copy is usually sufficient here as tuples are immutable
             return self._connections.copy()
@@ -571,7 +571,7 @@ class PipelineTopology:
             return len(self._stage_actors.get(stage_name, []))
 
     def get_edge_queues(self) -> Dict[str, Tuple[Any, int]]:
-        """Returns a shallow copy of the edge queues dictionary."""
+        """Returns a shallow copy of the edge queues' dictionary."""
         with self._lock:
             return self._edge_queues.copy()
 

nv_ingest/framework/orchestration/ray/primitives/ray_stat_collector.py

@@ -40,7 +40,7 @@ class RayStatsCollector:
             - `get_edge_queues() -> Dict[str, Tuple[Any, int]]`
             These methods should return snapshots suitable for iteration.
         interval : float, optional
-            The interval in seconds between stats collection attempts, by default 5.0.
+            The interval in seconds between stat collection attempts, by default 5.0.
         actor_timeout : float, optional
             Timeout in seconds for waiting for stats from a single actor, by default 5.0.
         queue_timeout : float, optional

nv_ingest/framework/orchestration/ray/stages/injectors/metadata_injector.py

@@ -2,6 +2,7 @@
 # All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+from datetime import datetime
 import logging
 import pandas as pd
 from typing import Any
@@ -9,8 +10,15 @@ from pydantic import BaseModel
 import ray
 
 from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
-from nv_ingest_api.internal.enums.common import DocumentTypeEnum, ContentTypeEnum
+from nv_ingest_api.internal.enums.common import (
+    DocumentTypeEnum,
+    ContentTypeEnum,
+    AccessLevelEnum,
+    TextTypeEnum,
+    LanguageEnum,
+)
 from nv_ingest_api.internal.primitives.tracing.tagging import traceable
+from nv_ingest_api.internal.schemas.meta.metadata_schema import ContentHierarchySchema
 from nv_ingest_api.util.converters.type_mappings import doc_type_to_content_type
 from nv_ingest_api.util.exception_handlers.decorators import (
     nv_ingest_node_failure_try_except,
@@ -61,27 +69,83 @@ class MetadataInjectionStage(RayActorStage):
                 # Convert document type to content type using enums.
                 content_type = doc_type_to_content_type(DocumentTypeEnum(row["document_type"]))
                 # Check if metadata is missing or doesn't contain 'content'
-                if "metadata" not in row or not isinstance(row["metadata"], dict) or "content" not in row["metadata"]:
+                if (
+                    "metadata" not in row
+                    or not isinstance(row["metadata"], dict)
+                    or "content" not in row["metadata"].keys()
+                ):
                     update_required = True
+
+                    # Initialize default structures based on MetaDataSchema
+                    default_source_metadata = {
+                        "source_id": row.get("source_id"),
+                        "source_name": row.get("source_name"),
+                        "source_type": row["document_type"],
+                        "source_location": "",
+                        "collection_id": "",
+                        "date_created": datetime.now().isoformat(),
+                        "last_modified": datetime.now().isoformat(),
+                        "summary": "",
+                        "partition_id": -1,
+                        "access_level": AccessLevelEnum.UNKNOWN.value,
+                    }
+
+                    default_content_metadata = {
+                        "type": content_type.name.lower(),
+                        "page_number": -1,
+                        "description": "",
+                        "hierarchy": ContentHierarchySchema().model_dump(),
+                        "subtype": "",
+                        "start_time": -1,
+                        "end_time": -1,
+                    }
+
+                    default_audio_metadata = None
+                    if content_type == ContentTypeEnum.AUDIO:
+                        default_audio_metadata = {
+                            "audio_type": row["document_type"],
+                            "audio_transcript": "",
+                        }
+
+                    default_image_metadata = None
+                    if content_type == ContentTypeEnum.IMAGE:
+                        default_image_metadata = {
+                            "image_type": row["document_type"],
+                            "structured_image_type": ContentTypeEnum.NONE.value,
+                            "caption": "",
+                            "text": "",
+                            "image_location": (0, 0, 0, 0),
+                            "image_location_max_dimensions": (0, 0),
+                            "uploaded_image_url": "",
+                            "width": 0,
+                            "height": 0,
+                        }
+
+                    default_text_metadata = None
+                    if content_type == ContentTypeEnum.TEXT:
+                        default_text_metadata = {
+                            "text_type": TextTypeEnum.DOCUMENT.value,
+                            "summary": "",
+                            "keywords": "",
+                            "language": LanguageEnum.UNKNOWN.value,
+                            "text_location": (0, 0, 0, 0),
+                            "text_location_max_dimensions": (0, 0, 0, 0),
+                        }
+
                     row["metadata"] = {
-                        "content": row.get("content"),
-                        "content_metadata": {
-                            "type": content_type.name.lower(),
-                        },
+                        "content": row["content"],
+                        "content_metadata": default_content_metadata,
                         "error_metadata": None,
-                        "audio_metadata": (
-                            None if content_type != ContentTypeEnum.AUDIO else {"audio_type": row["document_type"]}
-                        ),
-                        "image_metadata": (
-                            None if content_type != ContentTypeEnum.IMAGE else {"image_type": row["document_type"]}
-                        ),
-                        "source_metadata": {
-                            "source_id": row.get("source_id"),
-                            "source_name": row.get("source_name"),
-                            "source_type": row["document_type"],
-                        },
-                        "text_metadata": (None if content_type != ContentTypeEnum.TEXT else {"text_type": "document"}),
+                        "audio_metadata": default_audio_metadata,
+                        "image_metadata": default_image_metadata,
+                        "source_metadata": default_source_metadata,
+                        "text_metadata": default_text_metadata,
                     }
+                    logger.info(
+                        f"METADATA_INJECTOR_DEBUG: Rebuilt metadata for source_id='{row.get('source_id', 'N/A')}'. "
+                        f"Metadata keys: {list(row['metadata'].keys())}."
+                        f"'content' present: {'content' in row['metadata']}"
+                    )
             except Exception as inner_e:
                 logger.exception("Failed to process row during metadata injection")
                 raise inner_e

nv_ingest/framework/orchestration/ray/util/pipeline/pipeline_runners.py

@@ -331,6 +331,10 @@ def run_pipeline(
     """
     if run_in_subprocess:
         logger.info("Launching pipeline in Python subprocess using multiprocessing.")
+        if (ingest_config.ngc_api_key is None or ingest_config.ngc_api_key == "") and (
+            ingest_config.nvidia_build_api_key is None or ingest_config.nvidia_build_api_key == ""
+        ):
+            logger.warning("NGC_API_KEY or NVIDIA_BUILD_API_KEY are not set. NIM Related functions will not work.")
 
         ctx = multiprocessing.get_context("fork")
         process = ctx.Process(

nv_ingest/framework/orchestration/ray/util/pipeline/stage_builders.py

@@ -479,7 +479,7 @@ def add_image_caption_stage(pipeline, default_cpu_count, stage_name="image_capti
         **{
             "api_key": auth_token,
             "endpoint_url": endpoint_url,
-            "image_caption_model_name": model_name,
+            "model_name": model_name,
             "prompt": "Caption the content of this image:",
         }
     )

nv_ingest/version.py

@@ -5,7 +5,6 @@
 
 import datetime
 import os
-import re
 
 
 def get_version():
@@ -16,13 +15,6 @@ def get_version():
     if not version:
         version = f"{datetime.datetime.now().strftime('%Y.%m.%d')}"
 
-    # We only check this for dev, we assume for release the user knows what they are doing
-    if release_type != "release":
-        # Ensure the version is PEP 440 compatible
-        pep440_regex = r"^\d{4}\.\d{1,2}\.\d{1,2}$"
-        if not re.match(pep440_regex, version):
-            raise ValueError(f"Version '{version}' is not PEP 440 compatible")
-
     # Construct the final version string
     if release_type == "dev":
         # If rev is not specified and defaults to 0 lets create a more meaningful development

{nv_ingest-2025.6.2.dev20250602.dist-info → nv_ingest-2025.6.24.dev20250625.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.6.2.dev20250602
+Version: 2025.6.24.dev20250625
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License
@@ -240,13 +240,13 @@ Requires-Dist: python-docx>=1.1.2
 Requires-Dist: python-dotenv>=1.0.1
 Requires-Dist: python-pptx>=1.0.2
 Requires-Dist: prometheus-client
-Requires-Dist: torch==2.4.1
+Requires-Dist: torch>=2.4.1
 Requires-Dist: ray[all]>=2.37.0
 Requires-Dist: redis>=5.2.1
 Requires-Dist: requests>=2.28.2
 Requires-Dist: scikit-learn>=1.6.0
 Requires-Dist: scipy>=1.15.1
-Requires-Dist: setuptools>=58.2.0
+Requires-Dist: setuptools>=78.1.1
 Requires-Dist: tabulate>=0.9.0
 Requires-Dist: torchvision
 Requires-Dist: torchaudio
@@ -259,7 +259,7 @@ Requires-Dist: opencv-python
 Requires-Dist: pymilvus>=2.5.10
 Requires-Dist: pymilvus[bulk_writer,model]
 Requires-Dist: tritonclient
-Requires-Dist: nvidia-riva-client>=2.18.0
+Requires-Dist: nvidia-riva-client==2.20.0
 Requires-Dist: unstructured-client
 Requires-Dist: markitdown
 Dynamic: license-file

{nv_ingest-2025.6.2.dev20250602.dist-info → nv_ingest-2025.6.24.dev20250625.dist-info}/RECORD

@@ -1,5 +1,5 @@
 nv_ingest/__init__.py,sha256=vJLPeuxiIHqbxXPJSu9qe3MS-GPavbOUExyRq83DxxM,895
-nv_ingest/version.py,sha256=Y9gMjlV_tnRSE3JbmS1rWIfVppM974_g0k30MRF3IQM,1352
+nv_ingest/version.py,sha256=MG7DxlzpnoJI56vqxwzs9WeMAEI3uPhfDiNLs6GN6wI,986
 nv_ingest/api/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/api/main.py,sha256=XE-p4lJp1E7CCDOB8ENtYFrf63Dtq2bzQiGxpRfL2LA,1603
 nv_ingest/api/v1/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
@@ -14,15 +14,15 @@ nv_ingest/framework/orchestration/ray/edges/async_queue_edge.py,sha256=PQliU_kyG
 nv_ingest/framework/orchestration/ray/edges/ray_queue_edge.py,sha256=VFii2yxJuikimOxie3edKq5JN06g78AF8bdHSHVX8p8,2677
 nv_ingest/framework/orchestration/ray/edges/threaded_queue_edge.py,sha256=N6NH4KgZJ60e_JkGRcSmfQtX37qtX4TMcavOR-n3heE,2549
 nv_ingest/framework/orchestration/ray/examples/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/framework/orchestration/ray/examples/pipeline_test_harness.py,sha256=QHhoHjELAjs1SNGUOcUBU7f9JXxq3xTwM6fHyzcQytg,16450
+nv_ingest/framework/orchestration/ray/examples/pipeline_test_harness.py,sha256=QOwnxSCuhmkk-Ak-k5ILkSIqDdk5umJJy4rQloIHNMM,16408
 nv_ingest/framework/orchestration/ray/examples/task_source_harness.py,sha256=Yt7uxThg7s8WuMiaHLKC8r1XAG7QixegfkT-juE5oNw,1953
 nv_ingest/framework/orchestration/ray/examples/task_source_sink_harness.py,sha256=XkvsoIzH5ftXvAZ4ox7mxbx7ESVx6D8Xupcwbqgd52w,3277
 nv_ingest/framework/orchestration/ray/primitives/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/ray/primitives/dataclasses.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nv_ingest/framework/orchestration/ray/primitives/pipeline_monitor.py,sha256=L8ENPiF-lxqhIXVEQwQD5CCqQMb710ynj5D_Y4ixGhs,11077
-nv_ingest/framework/orchestration/ray/primitives/pipeline_topology.py,sha256=gc9gZNqPmnP76M-u8sQXyJd5aTSlyY_0CjLYNa-zvzk,29106
+nv_ingest/framework/orchestration/ray/primitives/pipeline_topology.py,sha256=2Xg7QoKKPPFUWkLck7NtEtb1xLnK3b5uUw8LRxPhLyw,29106
 nv_ingest/framework/orchestration/ray/primitives/ray_pipeline.py,sha256=BEBLjkYFXIH396EUQcfuxhrWlIMs9i6z7YfeeqJ5cZg,59579
-nv_ingest/framework/orchestration/ray/primitives/ray_stat_collector.py,sha256=yPIvOhxY42P-gf5dLkcPkfvfwL_I-ay0C8k5eNaU-VA,15811
+nv_ingest/framework/orchestration/ray/primitives/ray_stat_collector.py,sha256=AJ79OTh_NxxoTcyBNiopq3K_nLumsB9UU_axqQS3Gus,15810
 nv_ingest/framework/orchestration/ray/stages/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/ray/stages/extractors/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/ray/stages/extractors/audio_extractor.py,sha256=KV4hvY0NTGG8CjZviTgcFLQzaH8WJJGkkb9PFYbROww,3417
@@ -35,7 +35,7 @@ nv_ingest/framework/orchestration/ray/stages/extractors/pdf_extractor.py,sha256=
 nv_ingest/framework/orchestration/ray/stages/extractors/pptx_extractor.py,sha256=ywPGA-3GNsbp3FWFsu04foumM6ZCccRrm73ijS7oY0g,3581
 nv_ingest/framework/orchestration/ray/stages/extractors/table_extractor.py,sha256=EOcjyJYAB3TuXewZFld4shnGQUQ9VysjPrIWnmb8zuI,3893
 nv_ingest/framework/orchestration/ray/stages/injectors/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/framework/orchestration/ray/stages/injectors/metadata_injector.py,sha256=cdGbLBH0x-4uCdhr6JH_dMVFyBqPODQ5_WYO1otk8tI,4147
+nv_ingest/framework/orchestration/ray/stages/injectors/metadata_injector.py,sha256=K8jase7PD9kd8AuntzjdS1IO4ae8Oo_6byZsFG777D0,6838
 nv_ingest/framework/orchestration/ray/stages/meta/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/ray/stages/meta/ray_actor_edge_base.py,sha256=LnVqBJmpfCmcI-eJLbkwK-7SS-hpEp98P4iCRv_Zhb0,1726
 nv_ingest/framework/orchestration/ray/stages/meta/ray_actor_sink_stage_base.py,sha256=AhlZUbDK2Jckqnu8hVbJrckW8MsSixfmWc1bst9gRYk,3447
@@ -66,8 +66,8 @@ nv_ingest/framework/orchestration/ray/util/__init__.py,sha256=wQSlVx3T14ZgQAt-EP
 nv_ingest/framework/orchestration/ray/util/pipeline/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/ray/util/pipeline/pid_controller.py,sha256=AWyCFPP41vp1NOkO2urqm7vh-sTGKypJxwhdq8HxK6Q,50681
 nv_ingest/framework/orchestration/ray/util/pipeline/pipeline_builders.py,sha256=jMYnVe_0rb1OIO9mlB4LH3uXtgaXBbUG-rDPx6fe6J8,10456
-nv_ingest/framework/orchestration/ray/util/pipeline/pipeline_runners.py,sha256=3aSYSxyunm-eKUYErDArQTHXSoNKlNJMUr9o5Ui6VTk,14037
-nv_ingest/framework/orchestration/ray/util/pipeline/stage_builders.py,sha256=_MPUbOVTo9CjkBdDA--mcpu2plQ9qFY_TCBXbfpbB_A,21477
+nv_ingest/framework/orchestration/ray/util/pipeline/pipeline_runners.py,sha256=IKQHlEwe0xsjr4MgQJVL0UtnKha1qaoPFc08DF5QzMM,14351
+nv_ingest/framework/orchestration/ray/util/pipeline/stage_builders.py,sha256=ZFJkeJNbDM_GsedUlfk2B8kI93L_MNK6gxPgeryZM6I,21463
 nv_ingest/framework/orchestration/ray/util/system_tools/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/ray/util/system_tools/memory.py,sha256=ICqY0LLB3hFTZk03iX5yffMSKFH2q_aQomtDVzS_mKw,2228
 nv_ingest/framework/orchestration/ray/util/system_tools/visualizers.py,sha256=2oHZdO_3L1LGuzpyNmZBDh19n0E-APAaHk4MEwBwSHs,12895
@@ -95,8 +95,8 @@ nv_ingest/framework/util/service/meta/ingest/__init__.py,sha256=wQSlVx3T14ZgQAt-
 nv_ingest/framework/util/service/meta/ingest/ingest_service_meta.py,sha256=QS3uNxWBl5dIcmIpJKNe8_TLcTUuN2vcKyHeAwa-eSo,1589
 nv_ingest/framework/util/telemetry/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/util/telemetry/global_stats.py,sha256=nq65pEEdiwjAfGiqsxG1CeQMC96O3CfQxsZuGFCY-ds,4554
-nv_ingest-2025.6.2.dev20250602.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-nv_ingest-2025.6.2.dev20250602.dist-info/METADATA,sha256=PiLNsp7gjUH7ll310hQOEQCNk6i1Bf-7D7kg38MnLps,15141
-nv_ingest-2025.6.2.dev20250602.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nv_ingest-2025.6.2.dev20250602.dist-info/top_level.txt,sha256=sjb0ajIsgn3YgftSjZHlYO0HjYAIIhNuXG_AmywCvaU,10
-nv_ingest-2025.6.2.dev20250602.dist-info/RECORD,,
+nv_ingest-2025.6.24.dev20250625.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+nv_ingest-2025.6.24.dev20250625.dist-info/METADATA,sha256=jTy6k3k5HVq3vTjd8mr9e6xDYpTKH7Skd-SmXrCN9ws,15142
+nv_ingest-2025.6.24.dev20250625.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nv_ingest-2025.6.24.dev20250625.dist-info/top_level.txt,sha256=sjb0ajIsgn3YgftSjZHlYO0HjYAIIhNuXG_AmywCvaU,10
+nv_ingest-2025.6.24.dev20250625.dist-info/RECORD,,