nv-ingest 25.6.2__tar.gz → 25.6.26.dev20250626__tar.gz

{nv_ingest-25.6.2 → nv_ingest-25.6.26.dev20250626}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 25.6.2
+Version: 25.6.26.dev20250626
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License
@@ -229,7 +229,6 @@ Requires-Dist: openai>=1.82.0
 Requires-Dist: opentelemetry-api>=1.27.0
 Requires-Dist: opentelemetry-exporter-otlp>=1.27.0
 Requires-Dist: opentelemetry-sdk>=1.27.0
-Requires-Dist: nv-ingest-api==25.6.2
 Requires-Dist: pydantic>2.0.0
 Requires-Dist: pydantic-settings>2.0.0
 Requires-Dist: pypdfium2==4.30.1

{nv_ingest-25.6.2 → nv_ingest-25.6.26.dev20250626}/nv_ingest/api/main.py

@@ -22,7 +22,7 @@ logger = logging.getLogger(__name__)
 app = FastAPI(
     title="NV-Ingest Microservice",
     description="Service for ingesting heterogenous datatypes",
-    version="25.6.2",
+    version="25.4.2",
     contact={
         "name": "NVIDIA Corporation",
         "url": "https://nvidia.com",

{nv_ingest-25.6.2 → nv_ingest-25.6.26.dev20250626}/nv_ingest/framework/orchestration/ray/primitives/pipeline_topology.py

@@ -555,7 +555,7 @@ class PipelineTopology:
             return None
 
     def get_connections(self) -> Dict[str, List[Tuple[str, int]]]:
-        """Returns a shallow copy of the connections dictionary."""
+        """Returns a shallow copy of the connection dictionary."""
         with self._lock:
             # Shallow copy is usually sufficient here as tuples are immutable
             return self._connections.copy()
@@ -571,7 +571,7 @@ class PipelineTopology:
             return len(self._stage_actors.get(stage_name, []))
 
     def get_edge_queues(self) -> Dict[str, Tuple[Any, int]]:
-        """Returns a shallow copy of the edge queues dictionary."""
+        """Returns a shallow copy of the edge queues' dictionary."""
         with self._lock:
             return self._edge_queues.copy()
 

{nv_ingest-25.6.2 → nv_ingest-25.6.26.dev20250626}/nv_ingest/framework/orchestration/ray/primitives/ray_stat_collector.py

@@ -40,7 +40,7 @@ class RayStatsCollector:
             - `get_edge_queues() -> Dict[str, Tuple[Any, int]]`
             These methods should return snapshots suitable for iteration.
         interval : float, optional
-            The interval in seconds between stats collection attempts, by default 5.0.
+            The interval in seconds between stat collection attempts, by default 5.0.
         actor_timeout : float, optional
             Timeout in seconds for waiting for stats from a single actor, by default 5.0.
         queue_timeout : float, optional

nv_ingest-25.6.26.dev20250626/nv_ingest/framework/orchestration/ray/stages/injectors/metadata_injector.py

@@ -0,0 +1,161 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from datetime import datetime
+import logging
+import pandas as pd
+from typing import Any
+from pydantic import BaseModel
+import ray
+
+from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
+from nv_ingest_api.internal.enums.common import (
+    DocumentTypeEnum,
+    ContentTypeEnum,
+    AccessLevelEnum,
+    TextTypeEnum,
+    LanguageEnum,
+)
+from nv_ingest_api.internal.primitives.tracing.tagging import traceable
+from nv_ingest_api.internal.schemas.meta.metadata_schema import ContentHierarchySchema
+from nv_ingest_api.util.converters.type_mappings import doc_type_to_content_type
+from nv_ingest_api.util.exception_handlers.decorators import (
+    nv_ingest_node_failure_try_except,
+)
+
+# logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+
+@ray.remote
+class MetadataInjectionStage(RayActorStage):
+    """
+    A Ray actor stage that performs metadata injection on IngestControlMessages.
+
+    This stage iterates over the rows of the DataFrame payload, checks if metadata
+    injection is required, and if so, injects the appropriate metadata.
+    """
+
+    def __init__(self, config: BaseModel) -> None:
+        # Call the base initializer to set attributes like self._running.
+        super().__init__(config)
+        # Additional initialization can be added here if necessary.
+        logger.info("MetadataInjectionStage initialized with config: %s", config)
+
+    @traceable("metadata_injector")
+    @nv_ingest_node_failure_try_except(annotation_id="metadata_injector", raise_on_failure=False)
+    def on_data(self, message: Any) -> Any:
+        """
+        Process an incoming IngestControlMessage by injecting metadata into its DataFrame payload.
+
+        Parameters
+        ----------
+        message : IngestControlMessage
+            The incoming message containing the payload DataFrame.
+
+        Returns
+        -------
+        IngestControlMessage
+            The message with updated metadata if injection was required.
+        """
+        df = message.payload()
+        update_required = False
+        rows = []
+        logger.info("Starting metadata injection on DataFrame with %d rows", len(df))
+
+        for _, row in df.iterrows():
+            try:
+                # Convert document type to content type using enums.
+                content_type = doc_type_to_content_type(DocumentTypeEnum(row["document_type"]))
+                # Check if metadata is missing or doesn't contain 'content'
+                if (
+                    "metadata" not in row
+                    or not isinstance(row["metadata"], dict)
+                    or "content" not in row["metadata"].keys()
+                ):
+                    update_required = True
+
+                    # Initialize default structures based on MetaDataSchema
+                    default_source_metadata = {
+                        "source_id": row.get("source_id"),
+                        "source_name": row.get("source_name"),
+                        "source_type": row["document_type"],
+                        "source_location": "",
+                        "collection_id": "",
+                        "date_created": datetime.now().isoformat(),
+                        "last_modified": datetime.now().isoformat(),
+                        "summary": "",
+                        "partition_id": -1,
+                        "access_level": AccessLevelEnum.UNKNOWN.value,
+                    }
+
+                    default_content_metadata = {
+                        "type": content_type.name.lower(),
+                        "page_number": -1,
+                        "description": "",
+                        "hierarchy": ContentHierarchySchema().model_dump(),
+                        "subtype": "",
+                        "start_time": -1,
+                        "end_time": -1,
+                    }
+
+                    default_audio_metadata = None
+                    if content_type == ContentTypeEnum.AUDIO:
+                        default_audio_metadata = {
+                            "audio_type": row["document_type"],
+                            "audio_transcript": "",
+                        }
+
+                    default_image_metadata = None
+                    if content_type == ContentTypeEnum.IMAGE:
+                        default_image_metadata = {
+                            "image_type": row["document_type"],
+                            "structured_image_type": ContentTypeEnum.NONE.value,
+                            "caption": "",
+                            "text": "",
+                            "image_location": (0, 0, 0, 0),
+                            "image_location_max_dimensions": (0, 0),
+                            "uploaded_image_url": "",
+                            "width": 0,
+                            "height": 0,
+                        }
+
+                    default_text_metadata = None
+                    if content_type == ContentTypeEnum.TEXT:
+                        default_text_metadata = {
+                            "text_type": TextTypeEnum.DOCUMENT.value,
+                            "summary": "",
+                            "keywords": "",
+                            "language": LanguageEnum.UNKNOWN.value,
+                            "text_location": (0, 0, 0, 0),
+                            "text_location_max_dimensions": (0, 0, 0, 0),
+                        }
+
+                    row["metadata"] = {
+                        "content": row["content"],
+                        "content_metadata": default_content_metadata,
+                        "error_metadata": None,
+                        "audio_metadata": default_audio_metadata,
+                        "image_metadata": default_image_metadata,
+                        "source_metadata": default_source_metadata,
+                        "text_metadata": default_text_metadata,
+                    }
+                    logger.info(
+                        f"METADATA_INJECTOR_DEBUG: Rebuilt metadata for source_id='{row.get('source_id', 'N/A')}'. "
+                        f"Metadata keys: {list(row['metadata'].keys())}."
+                        f"'content' present: {'content' in row['metadata']}"
+                    )
+            except Exception as inner_e:
+                logger.exception("Failed to process row during metadata injection")
+                raise inner_e
+            rows.append(row)
+
+        if update_required:
+            docs = pd.DataFrame(rows)
+            message.payload(docs)
+            logger.info("Metadata injection updated payload with %d rows", len(docs))
+        else:
+            logger.info("No metadata update was necessary during metadata injection")
+
+        return message

{nv_ingest-25.6.2 → nv_ingest-25.6.26.dev20250626}/nv_ingest/version.py

@@ -5,7 +5,6 @@
 
 import datetime
 import os
-import re
 
 
 def get_version():
@@ -16,13 +15,6 @@ def get_version():
     if not version:
         version = f"{datetime.datetime.now().strftime('%Y.%m.%d')}"
 
-    # We only check this for dev, we assume for release the user knows what they are doing
-    if release_type != "release":
-        # Ensure the version is PEP 440 compatible
-        pep440_regex = r"^\d{4}\.\d{1,2}\.\d{1,2}$"
-        if not re.match(pep440_regex, version):
-            raise ValueError(f"Version '{version}' is not PEP 440 compatible")
-
     # Construct the final version string
     if release_type == "dev":
         # If rev is not specified and defaults to 0 lets create a more meaningful development

{nv_ingest-25.6.2 → nv_ingest-25.6.26.dev20250626}/nv_ingest.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 25.6.2
+Version: 25.6.26.dev20250626
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License
@@ -229,7 +229,6 @@ Requires-Dist: openai>=1.82.0
 Requires-Dist: opentelemetry-api>=1.27.0
 Requires-Dist: opentelemetry-exporter-otlp>=1.27.0
 Requires-Dist: opentelemetry-sdk>=1.27.0
-Requires-Dist: nv-ingest-api==25.6.2
 Requires-Dist: pydantic>2.0.0
 Requires-Dist: pydantic-settings>2.0.0
 Requires-Dist: pypdfium2==4.30.1

{nv_ingest-25.6.2 → nv_ingest-25.6.26.dev20250626}/nv_ingest.egg-info/requires.txt

@@ -14,7 +14,6 @@ openai>=1.82.0
 opentelemetry-api>=1.27.0
 opentelemetry-exporter-otlp>=1.27.0
 opentelemetry-sdk>=1.27.0
-nv-ingest-api==25.6.2
 pydantic>2.0.0
 pydantic-settings>2.0.0
 pypdfium2==4.30.1

{nv_ingest-25.6.2 → nv_ingest-25.6.26.dev20250626}/pyproject.toml

@@ -33,7 +33,6 @@ dependencies = [
     "opentelemetry-api>=1.27.0",
     "opentelemetry-exporter-otlp>=1.27.0",
     "opentelemetry-sdk>=1.27.0",
-    "nv-ingest-api==25.6.2",
     "pydantic>2.0.0",
     "pydantic-settings>2.0.0",
     "pypdfium2==4.30.1",

nv_ingest-25.6.2/nv_ingest/framework/orchestration/ray/stages/injectors/metadata_injector.py

@@ -1,97 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
-# All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-import logging
-import pandas as pd
-from typing import Any
-from pydantic import BaseModel
-import ray
-
-from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
-from nv_ingest_api.internal.enums.common import DocumentTypeEnum, ContentTypeEnum
-from nv_ingest_api.internal.primitives.tracing.tagging import traceable
-from nv_ingest_api.util.converters.type_mappings import doc_type_to_content_type
-from nv_ingest_api.util.exception_handlers.decorators import (
-    nv_ingest_node_failure_try_except,
-)
-
-# logging.basicConfig(level=logging.DEBUG)
-logger = logging.getLogger(__name__)
-
-
-@ray.remote
-class MetadataInjectionStage(RayActorStage):
-    """
-    A Ray actor stage that performs metadata injection on IngestControlMessages.
-
-    This stage iterates over the rows of the DataFrame payload, checks if metadata
-    injection is required, and if so, injects the appropriate metadata.
-    """
-
-    def __init__(self, config: BaseModel) -> None:
-        # Call the base initializer to set attributes like self._running.
-        super().__init__(config)
-        # Additional initialization can be added here if necessary.
-        logger.info("MetadataInjectionStage initialized with config: %s", config)
-
-    @traceable("metadata_injector")
-    @nv_ingest_node_failure_try_except(annotation_id="metadata_injector", raise_on_failure=False)
-    def on_data(self, message: Any) -> Any:
-        """
-        Process an incoming IngestControlMessage by injecting metadata into its DataFrame payload.
-
-        Parameters
-        ----------
-        message : IngestControlMessage
-            The incoming message containing the payload DataFrame.
-
-        Returns
-        -------
-        IngestControlMessage
-            The message with updated metadata if injection was required.
-        """
-        df = message.payload()
-        update_required = False
-        rows = []
-        logger.info("Starting metadata injection on DataFrame with %d rows", len(df))
-
-        for _, row in df.iterrows():
-            try:
-                # Convert document type to content type using enums.
-                content_type = doc_type_to_content_type(DocumentTypeEnum(row["document_type"]))
-                # Check if metadata is missing or doesn't contain 'content'
-                if "metadata" not in row or not isinstance(row["metadata"], dict) or "content" not in row["metadata"]:
-                    update_required = True
-                    row["metadata"] = {
-                        "content": row.get("content"),
-                        "content_metadata": {
-                            "type": content_type.name.lower(),
-                        },
-                        "error_metadata": None,
-                        "audio_metadata": (
-                            None if content_type != ContentTypeEnum.AUDIO else {"audio_type": row["document_type"]}
-                        ),
-                        "image_metadata": (
-                            None if content_type != ContentTypeEnum.IMAGE else {"image_type": row["document_type"]}
-                        ),
-                        "source_metadata": {
-                            "source_id": row.get("source_id"),
-                            "source_name": row.get("source_name"),
-                            "source_type": row["document_type"],
-                        },
-                        "text_metadata": (None if content_type != ContentTypeEnum.TEXT else {"text_type": "document"}),
-                    }
-            except Exception as inner_e:
-                logger.exception("Failed to process row during metadata injection")
-                raise inner_e
-            rows.append(row)
-
-        if update_required:
-            docs = pd.DataFrame(rows)
-            message.payload(docs)
-            logger.info("Metadata injection updated payload with %d rows", len(docs))
-        else:
-            logger.info("No metadata update was necessary during metadata injection")
-
-        return message