nv-ingest 2025.6.2.dev20250602__tar.gz → 2025.6.24.dev20250625__tar.gz

{nv_ingest-2025.6.2.dev20250602 → nv_ingest-2025.6.24.dev20250625}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.6.2.dev20250602
+Version: 2025.6.24.dev20250625
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License
@@ -240,13 +240,13 @@ Requires-Dist: python-docx>=1.1.2
 Requires-Dist: python-dotenv>=1.0.1
 Requires-Dist: python-pptx>=1.0.2
 Requires-Dist: prometheus-client
-Requires-Dist: torch==2.4.1
+Requires-Dist: torch>=2.4.1
 Requires-Dist: ray[all]>=2.37.0
 Requires-Dist: redis>=5.2.1
 Requires-Dist: requests>=2.28.2
 Requires-Dist: scikit-learn>=1.6.0
 Requires-Dist: scipy>=1.15.1
-Requires-Dist: setuptools>=58.2.0
+Requires-Dist: setuptools>=78.1.1
 Requires-Dist: tabulate>=0.9.0
 Requires-Dist: torchvision
 Requires-Dist: torchaudio
@@ -259,7 +259,7 @@ Requires-Dist: opencv-python
 Requires-Dist: pymilvus>=2.5.10
 Requires-Dist: pymilvus[bulk_writer,model]
 Requires-Dist: tritonclient
-Requires-Dist: nvidia-riva-client>=2.18.0
+Requires-Dist: nvidia-riva-client==2.20.0
 Requires-Dist: unstructured-client
 Requires-Dist: markitdown
 Dynamic: license-file

{nv_ingest-2025.6.2.dev20250602 → nv_ingest-2025.6.24.dev20250625}/nv_ingest/framework/orchestration/ray/examples/pipeline_test_harness.py

@@ -155,7 +155,7 @@ if __name__ == "__main__":
     logger.info("Environment variables set.")
 
     image_caption_endpoint_url = "https://integrate.api.nvidia.com/v1/chat/completions"
-    image_caption_model_name = "meta/llama-3.2-11b-vision-instruct"
+    model_name = "meta/llama-3.2-11b-vision-instruct"
     yolox_grpc, yolox_http, yolox_auth, yolox_protocol = get_nim_service("yolox")
     (
         yolox_table_structure_grpc,
@@ -228,7 +228,7 @@ if __name__ == "__main__":
     image_caption_config = {
         "api_key": yolox_auth,
         "endpoint_url": image_caption_endpoint_url,
-        "image_caption_model_name": image_caption_model_name,
+        "model_name": model_name,
         "prompt": "Caption the content of this image:",
     }
     logger.info("Service configuration retrieved from get_nim_service and environment variables.")

{nv_ingest-2025.6.2.dev20250602 → nv_ingest-2025.6.24.dev20250625}/nv_ingest/framework/orchestration/ray/primitives/pipeline_topology.py

@@ -555,7 +555,7 @@ class PipelineTopology:
             return None
 
     def get_connections(self) -> Dict[str, List[Tuple[str, int]]]:
-        """Returns a shallow copy of the connections dictionary."""
+        """Returns a shallow copy of the connection dictionary."""
         with self._lock:
             # Shallow copy is usually sufficient here as tuples are immutable
             return self._connections.copy()
@@ -571,7 +571,7 @@ class PipelineTopology:
             return len(self._stage_actors.get(stage_name, []))
 
     def get_edge_queues(self) -> Dict[str, Tuple[Any, int]]:
-        """Returns a shallow copy of the edge queues dictionary."""
+        """Returns a shallow copy of the edge queues' dictionary."""
         with self._lock:
             return self._edge_queues.copy()
 

{nv_ingest-2025.6.2.dev20250602 → nv_ingest-2025.6.24.dev20250625}/nv_ingest/framework/orchestration/ray/primitives/ray_stat_collector.py

@@ -40,7 +40,7 @@ class RayStatsCollector:
             - `get_edge_queues() -> Dict[str, Tuple[Any, int]]`
             These methods should return snapshots suitable for iteration.
         interval : float, optional
-            The interval in seconds between stats collection attempts, by default 5.0.
+            The interval in seconds between stat collection attempts, by default 5.0.
         actor_timeout : float, optional
             Timeout in seconds for waiting for stats from a single actor, by default 5.0.
         queue_timeout : float, optional

nv_ingest-2025.6.24.dev20250625/nv_ingest/framework/orchestration/ray/stages/injectors/metadata_injector.py

@@ -0,0 +1,161 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from datetime import datetime
+import logging
+import pandas as pd
+from typing import Any
+from pydantic import BaseModel
+import ray
+
+from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
+from nv_ingest_api.internal.enums.common import (
+    DocumentTypeEnum,
+    ContentTypeEnum,
+    AccessLevelEnum,
+    TextTypeEnum,
+    LanguageEnum,
+)
+from nv_ingest_api.internal.primitives.tracing.tagging import traceable
+from nv_ingest_api.internal.schemas.meta.metadata_schema import ContentHierarchySchema
+from nv_ingest_api.util.converters.type_mappings import doc_type_to_content_type
+from nv_ingest_api.util.exception_handlers.decorators import (
+    nv_ingest_node_failure_try_except,
+)
+
+# logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+
+@ray.remote
+class MetadataInjectionStage(RayActorStage):
+    """
+    A Ray actor stage that performs metadata injection on IngestControlMessages.
+
+    This stage iterates over the rows of the DataFrame payload, checks if metadata
+    injection is required, and if so, injects the appropriate metadata.
+    """
+
+    def __init__(self, config: BaseModel) -> None:
+        # Call the base initializer to set attributes like self._running.
+        super().__init__(config)
+        # Additional initialization can be added here if necessary.
+        logger.info("MetadataInjectionStage initialized with config: %s", config)
+
+    @traceable("metadata_injector")
+    @nv_ingest_node_failure_try_except(annotation_id="metadata_injector", raise_on_failure=False)
+    def on_data(self, message: Any) -> Any:
+        """
+        Process an incoming IngestControlMessage by injecting metadata into its DataFrame payload.
+
+        Parameters
+        ----------
+        message : IngestControlMessage
+            The incoming message containing the payload DataFrame.
+
+        Returns
+        -------
+        IngestControlMessage
+            The message with updated metadata if injection was required.
+        """
+        df = message.payload()
+        update_required = False
+        rows = []
+        logger.info("Starting metadata injection on DataFrame with %d rows", len(df))
+
+        for _, row in df.iterrows():
+            try:
+                # Convert document type to content type using enums.
+                content_type = doc_type_to_content_type(DocumentTypeEnum(row["document_type"]))
+                # Check if metadata is missing or doesn't contain 'content'
+                if (
+                    "metadata" not in row
+                    or not isinstance(row["metadata"], dict)
+                    or "content" not in row["metadata"].keys()
+                ):
+                    update_required = True
+
+                    # Initialize default structures based on MetaDataSchema
+                    default_source_metadata = {
+                        "source_id": row.get("source_id"),
+                        "source_name": row.get("source_name"),
+                        "source_type": row["document_type"],
+                        "source_location": "",
+                        "collection_id": "",
+                        "date_created": datetime.now().isoformat(),
+                        "last_modified": datetime.now().isoformat(),
+                        "summary": "",
+                        "partition_id": -1,
+                        "access_level": AccessLevelEnum.UNKNOWN.value,
+                    }
+
+                    default_content_metadata = {
+                        "type": content_type.name.lower(),
+                        "page_number": -1,
+                        "description": "",
+                        "hierarchy": ContentHierarchySchema().model_dump(),
+                        "subtype": "",
+                        "start_time": -1,
+                        "end_time": -1,
+                    }
+
+                    default_audio_metadata = None
+                    if content_type == ContentTypeEnum.AUDIO:
+                        default_audio_metadata = {
+                            "audio_type": row["document_type"],
+                            "audio_transcript": "",
+                        }
+
+                    default_image_metadata = None
+                    if content_type == ContentTypeEnum.IMAGE:
+                        default_image_metadata = {
+                            "image_type": row["document_type"],
+                            "structured_image_type": ContentTypeEnum.NONE.value,
+                            "caption": "",
+                            "text": "",
+                            "image_location": (0, 0, 0, 0),
+                            "image_location_max_dimensions": (0, 0),
+                            "uploaded_image_url": "",
+                            "width": 0,
+                            "height": 0,
+                        }
+
+                    default_text_metadata = None
+                    if content_type == ContentTypeEnum.TEXT:
+                        default_text_metadata = {
+                            "text_type": TextTypeEnum.DOCUMENT.value,
+                            "summary": "",
+                            "keywords": "",
+                            "language": LanguageEnum.UNKNOWN.value,
+                            "text_location": (0, 0, 0, 0),
+                            "text_location_max_dimensions": (0, 0, 0, 0),
+                        }
+
+                    row["metadata"] = {
+                        "content": row["content"],
+                        "content_metadata": default_content_metadata,
+                        "error_metadata": None,
+                        "audio_metadata": default_audio_metadata,
+                        "image_metadata": default_image_metadata,
+                        "source_metadata": default_source_metadata,
+                        "text_metadata": default_text_metadata,
+                    }
+                    logger.info(
+                        f"METADATA_INJECTOR_DEBUG: Rebuilt metadata for source_id='{row.get('source_id', 'N/A')}'. "
+                        f"Metadata keys: {list(row['metadata'].keys())}."
+                        f"'content' present: {'content' in row['metadata']}"
+                    )
+            except Exception as inner_e:
+                logger.exception("Failed to process row during metadata injection")
+                raise inner_e
+            rows.append(row)
+
+        if update_required:
+            docs = pd.DataFrame(rows)
+            message.payload(docs)
+            logger.info("Metadata injection updated payload with %d rows", len(docs))
+        else:
+            logger.info("No metadata update was necessary during metadata injection")
+
+        return message

{nv_ingest-2025.6.2.dev20250602 → nv_ingest-2025.6.24.dev20250625}/nv_ingest/framework/orchestration/ray/util/pipeline/pipeline_runners.py

@@ -331,6 +331,10 @@ def run_pipeline(
     """
     if run_in_subprocess:
         logger.info("Launching pipeline in Python subprocess using multiprocessing.")
+        if (ingest_config.ngc_api_key is None or ingest_config.ngc_api_key == "") and (
+            ingest_config.nvidia_build_api_key is None or ingest_config.nvidia_build_api_key == ""
+        ):
+            logger.warning("NGC_API_KEY or NVIDIA_BUILD_API_KEY are not set. NIM Related functions will not work.")
 
         ctx = multiprocessing.get_context("fork")
         process = ctx.Process(

{nv_ingest-2025.6.2.dev20250602 → nv_ingest-2025.6.24.dev20250625}/nv_ingest/framework/orchestration/ray/util/pipeline/stage_builders.py

@@ -479,7 +479,7 @@ def add_image_caption_stage(pipeline, default_cpu_count, stage_name="image_capti
         **{
             "api_key": auth_token,
             "endpoint_url": endpoint_url,
-            "image_caption_model_name": model_name,
+            "model_name": model_name,
             "prompt": "Caption the content of this image:",
         }
     )

{nv_ingest-2025.6.2.dev20250602 → nv_ingest-2025.6.24.dev20250625}/nv_ingest/version.py

@@ -5,7 +5,6 @@
 
 import datetime
 import os
-import re
 
 
 def get_version():
@@ -16,13 +15,6 @@ def get_version():
     if not version:
         version = f"{datetime.datetime.now().strftime('%Y.%m.%d')}"
 
-    # We only check this for dev, we assume for release the user knows what they are doing
-    if release_type != "release":
-        # Ensure the version is PEP 440 compatible
-        pep440_regex = r"^\d{4}\.\d{1,2}\.\d{1,2}$"
-        if not re.match(pep440_regex, version):
-            raise ValueError(f"Version '{version}' is not PEP 440 compatible")
-
     # Construct the final version string
     if release_type == "dev":
         # If rev is not specified and defaults to 0 lets create a more meaningful development

{nv_ingest-2025.6.2.dev20250602 → nv_ingest-2025.6.24.dev20250625}/nv_ingest.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.6.2.dev20250602
+Version: 2025.6.24.dev20250625
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License
@@ -240,13 +240,13 @@ Requires-Dist: python-docx>=1.1.2
 Requires-Dist: python-dotenv>=1.0.1
 Requires-Dist: python-pptx>=1.0.2
 Requires-Dist: prometheus-client
-Requires-Dist: torch==2.4.1
+Requires-Dist: torch>=2.4.1
 Requires-Dist: ray[all]>=2.37.0
 Requires-Dist: redis>=5.2.1
 Requires-Dist: requests>=2.28.2
 Requires-Dist: scikit-learn>=1.6.0
 Requires-Dist: scipy>=1.15.1
-Requires-Dist: setuptools>=58.2.0
+Requires-Dist: setuptools>=78.1.1
 Requires-Dist: tabulate>=0.9.0
 Requires-Dist: torchvision
 Requires-Dist: torchaudio
@@ -259,7 +259,7 @@ Requires-Dist: opencv-python
 Requires-Dist: pymilvus>=2.5.10
 Requires-Dist: pymilvus[bulk_writer,model]
 Requires-Dist: tritonclient
-Requires-Dist: nvidia-riva-client>=2.18.0
+Requires-Dist: nvidia-riva-client==2.20.0
 Requires-Dist: unstructured-client
 Requires-Dist: markitdown
 Dynamic: license-file

{nv_ingest-2025.6.2.dev20250602 → nv_ingest-2025.6.24.dev20250625}/nv_ingest.egg-info/requires.txt

@@ -25,13 +25,13 @@ python-docx>=1.1.2
 python-dotenv>=1.0.1
 python-pptx>=1.0.2
 prometheus-client
-torch==2.4.1
+torch>=2.4.1
 ray[all]>=2.37.0
 redis>=5.2.1
 requests>=2.28.2
 scikit-learn>=1.6.0
 scipy>=1.15.1
-setuptools>=58.2.0
+setuptools>=78.1.1
 tabulate>=0.9.0
 torchvision
 torchaudio
@@ -44,6 +44,6 @@ opencv-python
 pymilvus>=2.5.10
 pymilvus[bulk_writer,model]
 tritonclient
-nvidia-riva-client>=2.18.0
+nvidia-riva-client==2.20.0
 unstructured-client
 markitdown

{nv_ingest-2025.6.2.dev20250602 → nv_ingest-2025.6.24.dev20250625}/pyproject.toml

@@ -44,13 +44,13 @@ dependencies = [
     "python-dotenv>=1.0.1",
     "python-pptx>=1.0.2",
     "prometheus-client",
-    "torch==2.4.1",
+    "torch>=2.4.1",
     "ray[all]>=2.37.0",
     "redis>=5.2.1",
     "requests>=2.28.2",
     "scikit-learn>=1.6.0",
     "scipy>=1.15.1",
-    "setuptools>=58.2.0",
+    "setuptools>=78.1.1",
     "tabulate>=0.9.0",
     "torchvision",
     "torchaudio",
@@ -63,7 +63,7 @@ dependencies = [
     "pymilvus>=2.5.10",
     "pymilvus[bulk_writer, model]",
     "tritonclient",
-    "nvidia-riva-client>=2.18.0",
+    "nvidia-riva-client==2.20.0",
     "unstructured-client",
     "markitdown",
 ]

nv_ingest-2025.6.2.dev20250602/nv_ingest/framework/orchestration/ray/stages/injectors/metadata_injector.py

@@ -1,97 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
-# All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-import logging
-import pandas as pd
-from typing import Any
-from pydantic import BaseModel
-import ray
-
-from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
-from nv_ingest_api.internal.enums.common import DocumentTypeEnum, ContentTypeEnum
-from nv_ingest_api.internal.primitives.tracing.tagging import traceable
-from nv_ingest_api.util.converters.type_mappings import doc_type_to_content_type
-from nv_ingest_api.util.exception_handlers.decorators import (
-    nv_ingest_node_failure_try_except,
-)
-
-# logging.basicConfig(level=logging.DEBUG)
-logger = logging.getLogger(__name__)
-
-
-@ray.remote
-class MetadataInjectionStage(RayActorStage):
-    """
-    A Ray actor stage that performs metadata injection on IngestControlMessages.
-
-    This stage iterates over the rows of the DataFrame payload, checks if metadata
-    injection is required, and if so, injects the appropriate metadata.
-    """
-
-    def __init__(self, config: BaseModel) -> None:
-        # Call the base initializer to set attributes like self._running.
-        super().__init__(config)
-        # Additional initialization can be added here if necessary.
-        logger.info("MetadataInjectionStage initialized with config: %s", config)
-
-    @traceable("metadata_injector")
-    @nv_ingest_node_failure_try_except(annotation_id="metadata_injector", raise_on_failure=False)
-    def on_data(self, message: Any) -> Any:
-        """
-        Process an incoming IngestControlMessage by injecting metadata into its DataFrame payload.
-
-        Parameters
-        ----------
-        message : IngestControlMessage
-            The incoming message containing the payload DataFrame.
-
-        Returns
-        -------
-        IngestControlMessage
-            The message with updated metadata if injection was required.
-        """
-        df = message.payload()
-        update_required = False
-        rows = []
-        logger.info("Starting metadata injection on DataFrame with %d rows", len(df))
-
-        for _, row in df.iterrows():
-            try:
-                # Convert document type to content type using enums.
-                content_type = doc_type_to_content_type(DocumentTypeEnum(row["document_type"]))
-                # Check if metadata is missing or doesn't contain 'content'
-                if "metadata" not in row or not isinstance(row["metadata"], dict) or "content" not in row["metadata"]:
-                    update_required = True
-                    row["metadata"] = {
-                        "content": row.get("content"),
-                        "content_metadata": {
-                            "type": content_type.name.lower(),
-                        },
-                        "error_metadata": None,
-                        "audio_metadata": (
-                            None if content_type != ContentTypeEnum.AUDIO else {"audio_type": row["document_type"]}
-                        ),
-                        "image_metadata": (
-                            None if content_type != ContentTypeEnum.IMAGE else {"image_type": row["document_type"]}
-                        ),
-                        "source_metadata": {
-                            "source_id": row.get("source_id"),
-                            "source_name": row.get("source_name"),
-                            "source_type": row["document_type"],
-                        },
-                        "text_metadata": (None if content_type != ContentTypeEnum.TEXT else {"text_type": "document"}),
-                    }
-            except Exception as inner_e:
-                logger.exception("Failed to process row during metadata injection")
-                raise inner_e
-            rows.append(row)
-
-        if update_required:
-            docs = pd.DataFrame(rows)
-            message.payload(docs)
-            logger.info("Metadata injection updated payload with %d rows", len(docs))
-        else:
-            logger.info("No metadata update was necessary during metadata injection")
-
-        return message