nv-ingest-api 2025.5.13.dev20250513__tar.gz → 2025.5.15.dev20250515__tar.gz

{nv_ingest_api-2025.5.13.dev20250513/src/nv_ingest_api.egg-info → nv_ingest_api-2025.5.15.dev20250515}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-api
-Version: 2025.5.13.dev20250513
+Version: 2025.5.15.dev20250515
 Summary: Python module with core document ingestion functions.
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest_api-2025.5.13.dev20250513 → nv_ingest_api-2025.5.15.dev20250515}/src/nv_ingest_api/internal/extract/image/chart_extractor.py

@@ -27,7 +27,7 @@ from nv_ingest_api.util.nim import create_inference_client
 PADDLE_MIN_WIDTH = 32
 PADDLE_MIN_HEIGHT = 32
 
-logger = logging.getLogger(f"morpheus.{__name__}")
+logger = logging.getLogger(f"ray.{__name__}")
 
 
 def _filter_valid_chart_images(
@@ -80,7 +80,7 @@ def _run_chart_inference(
             yolox_client.infer,
             data=data_yolox,
             model_name="yolox",
-            stage_name="chart_data_extraction",
+            stage_name="chart_extraction",
             max_batch_size=8,
             trace_info=trace_info,
         )
@@ -88,7 +88,7 @@ def _run_chart_inference(
             paddle_client.infer,
             data=data_paddle,
             model_name="paddle",
-            stage_name="chart_data_extraction",
+            stage_name="chart_extraction",
             max_batch_size=1 if paddle_client.protocol == "grpc" else 2,
             trace_info=trace_info,
         )

{nv_ingest_api-2025.5.13.dev20250513 → nv_ingest_api-2025.5.15.dev20250515}/src/nv_ingest_api/internal/extract/image/image_helpers/common.py

@@ -223,7 +223,7 @@ def extract_page_elements_from_images(
             model_name="yolox",
             max_batch_size=YOLOX_MAX_BATCH_SIZE,
             trace_info=trace_info,
-            stage_name="pdf_content_extractor",
+            stage_name="pdf_extraction",
         )
 
         # Process each result along with its corresponding image.

{nv_ingest_api-2025.5.13.dev20250513 → nv_ingest_api-2025.5.15.dev20250515}/src/nv_ingest_api/internal/extract/image/infographic_extractor.py

@@ -100,7 +100,7 @@ def _update_infographic_metadata(
         paddle_results = paddle_client.infer(
             data=data_paddle,
             model_name="paddle",
-            stage_name="infographic_data_extraction",
+            stage_name="infographic_extraction",
             max_batch_size=1 if paddle_client.protocol == "grpc" else 2,
             trace_info=trace_info,
         )

{nv_ingest_api-2025.5.13.dev20250513 → nv_ingest_api-2025.5.15.dev20250515}/src/nv_ingest_api/internal/extract/image/table_extractor.py

@@ -81,7 +81,7 @@ def _run_inference(
                 yolox_client.infer,
                 data=data_yolox,
                 model_name="yolox",
-                stage_name="table_data_extraction",
+                stage_name="table_extraction",
                 max_batch_size=8,
                 trace_info=trace_info,
             )
@@ -89,7 +89,7 @@ def _run_inference(
             paddle_client.infer,
             data=data_paddle,
             model_name="paddle",
-            stage_name="table_data_extraction",
+            stage_name="table_extraction",
             max_batch_size=1 if paddle_client.protocol == "grpc" else 2,
             trace_info=trace_info,
         )

{nv_ingest_api-2025.5.13.dev20250513 → nv_ingest_api-2025.5.15.dev20250515}/src/nv_ingest_api/internal/extract/pdf/engines/nemoretriever.py

@@ -466,7 +466,7 @@ def _extract_text_and_bounding_boxes(
     inference_results = nemoretriever_parse_client.infer(
         data=data,
         model_name="nemoretriever_parse",
-        stage_name="pdf_content_extractor",
+        stage_name="pdf_extraction",
         max_batch_size=NEMORETRIEVER_PARSE_MAX_BATCH_SIZE,
         execution_trace_log=execution_trace_log,
     )

{nv_ingest_api-2025.5.13.dev20250513 → nv_ingest_api-2025.5.15.dev20250515}/src/nv_ingest_api/internal/extract/pdf/engines/pdfium.py

@@ -105,7 +105,7 @@ def _extract_page_elements_using_image_ensemble(
             model_name="yolox",
             max_batch_size=YOLOX_MAX_BATCH_SIZE,
             trace_info=execution_trace_log,
-            stage_name="pdf_content_extractor",
+            stage_name="pdf_extraction",
         )
 
         # Process results: iterate over each image's inference output.

{nv_ingest_api-2025.5.13.dev20250513 → nv_ingest_api-2025.5.15.dev20250515}/src/nv_ingest_api/internal/extract/pptx/pptx_extractor.py

@@ -99,14 +99,11 @@ def _decode_and_extract_from_pptx(
 
     # Retrieve extraction parameters (and remove boolean flags as they are consumed).
     extract_params: Dict[str, Any] = prepared_task_props.get("params", {})
-    try:
-        extract_text: bool = extract_params.pop("extract_text", False)
-        extract_images: bool = extract_params.pop("extract_images", False)
-        extract_tables: bool = extract_params.pop("extract_tables", False)
-        extract_charts: bool = extract_params.pop("extract_charts", False)
-        extract_infographics: bool = extract_params.pop("extract_infographics", False)
-    except KeyError as e:
-        raise ValueError(f"Missing required extraction flag: {e}")
+    extract_text: bool = extract_params.pop("extract_text", False)
+    extract_images: bool = extract_params.pop("extract_images", False)
+    extract_tables: bool = extract_params.pop("extract_tables", False)
+    extract_charts: bool = extract_params.pop("extract_charts", False)
+    extract_infographics: bool = extract_params.pop("extract_infographics", False)
 
     # Inject additional configuration and trace information.
     if getattr(extraction_config, "pptx_extraction_config", None) is not None:

{nv_ingest_api-2025.5.13.dev20250513 → nv_ingest_api-2025.5.15.dev20250515}/src/nv_ingest_api/internal/schemas/extract/extract_chart_schema.py

@@ -129,7 +129,7 @@ class ChartExtractorSchema(BaseModel):
     @field_validator("max_queue_size", "n_workers")
     def check_positive(cls, v, field):
         if v <= 0:
-            raise ValueError(f"{field.field_name} must be greater than 10.")
+            raise ValueError(f"{field.field_name} must be greater than 0.")
         return v
 
     model_config = ConfigDict(extra="forbid")

{nv_ingest_api-2025.5.13.dev20250513 → nv_ingest_api-2025.5.15.dev20250515}/src/nv_ingest_api/internal/schemas/extract/extract_infographic_schema.py

@@ -122,7 +122,7 @@ class InfographicExtractorSchema(BaseModel):
     @field_validator("max_queue_size", "n_workers")
     def check_positive(cls, v, field):
         if v <= 0:
-            raise ValueError(f"{field.field_name} must be greater than 10.")
+            raise ValueError(f"{field.field_name} must be greater than 0.")
         return v
 
     model_config = ConfigDict(extra="forbid")

{nv_ingest_api-2025.5.13.dev20250513 → nv_ingest_api-2025.5.15.dev20250515}/src/nv_ingest_api/internal/schemas/extract/extract_table_schema.py

@@ -122,7 +122,7 @@ class TableExtractorSchema(BaseModel):
     @field_validator("max_queue_size", "n_workers")
     def check_positive(cls, v, field):
         if v <= 0:
-            raise ValueError(f"{field.field_name} must be greater than 10.")
+            raise ValueError(f"{field.field_name} must be greater than 0.")
         return v
 
     endpoint_config: Optional[TableExtractorConfigSchema] = None

nv_ingest_api-2025.5.15.dev20250515/src/nv_ingest_api/internal/schemas/message_brokers/message_broker_client_schema.py

@@ -0,0 +1,37 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from pydantic import BaseModel, Field
+from typing import Optional, Literal, Annotated
+
+
+class MessageBrokerClientSchema(BaseModel):
+    """
+    Configuration schema for message broker client connections.
+    Supports Redis or simple in-memory clients.
+    """
+
+    host: str = Field(default="redis", description="Hostname of the broker service.")
+
+    port: Annotated[int, Field(gt=0, lt=65536)] = Field(
+        default=6379, description="Port to connect to. Must be between 1 and 65535."
+    )
+
+    client_type: Literal["redis", "simple"] = Field(
+        default="redis", description="Type of broker client. Supported values: 'redis', 'simple'."
+    )
+
+    broker_params: Optional[dict] = Field(
+        default_factory=dict, description="Optional parameters passed to the broker client."
+    )
+
+    connection_timeout: Annotated[int, Field(ge=0)] = Field(
+        default=300, description="Connection timeout in seconds. Must be >= 0."
+    )
+
+    max_backoff: Annotated[int, Field(ge=0)] = Field(
+        default=300, description="Maximum backoff time in seconds. Must be >= 0."
+    )
+
+    max_retries: Annotated[int, Field(ge=0)] = Field(default=0, description="Maximum number of retries. Must be >= 0.")

{nv_ingest_api-2025.5.13.dev20250513 → nv_ingest_api-2025.5.15.dev20250515}/src/nv_ingest_api/internal/schemas/meta/ingest_job_schema.py

@@ -160,29 +160,40 @@ class IngestTaskSchema(BaseModelNoExt):
     @model_validator(mode="before")
     @classmethod
     def check_task_properties_type(cls, values):
-        task_type, task_properties = values.get("type"), values.get("task_properties", {})
-        if task_type and task_properties:
-            expected_type = {
-                TaskTypeEnum.CAPTION: IngestTaskCaptionSchema,
-                TaskTypeEnum.DEDUP: IngestTaskDedupSchema,
-                TaskTypeEnum.EMBED: IngestTaskEmbedSchema,
-                TaskTypeEnum.EXTRACT: IngestTaskExtractSchema,
-                TaskTypeEnum.FILTER: IngestTaskFilterSchema,  # Extend mapping as necessary
-                TaskTypeEnum.SPLIT: IngestTaskSplitSchema,
-                TaskTypeEnum.STORE_EMBEDDING: IngestTaskStoreEmbedSchema,
-                TaskTypeEnum.STORE: IngestTaskStoreSchema,
-                TaskTypeEnum.VDB_UPLOAD: IngestTaskVdbUploadSchema,
-                TaskTypeEnum.AUDIO_DATA_EXTRACT: IngestTaskAudioExtraction,
-                TaskTypeEnum.TABLE_DATA_EXTRACT: IngestTaskTableExtraction,
-                TaskTypeEnum.CHART_DATA_EXTRACT: IngestTaskChartExtraction,
-                TaskTypeEnum.INFOGRAPHIC_DATA_EXTRACT: IngestTaskInfographicExtraction,
-            }.get(
-                task_type
-            )  # Removed .upper()
-
-            # Validate task_properties against the expected schema.
-            validated_task_properties = expected_type(**task_properties)
-            values["task_properties"] = validated_task_properties
+        task_type = values.get("type")
+        task_properties = values.get("task_properties", {})
+
+        # Ensure task_type is lowercased and converted to enum early
+        if isinstance(task_type, str):
+            task_type = task_type.lower()
+            try:
+                task_type = TaskTypeEnum(task_type)
+            except ValueError:
+                raise ValueError(f"{task_type} is not a valid TaskTypeEnum value")
+
+        task_type_to_schema = {
+            TaskTypeEnum.CAPTION: IngestTaskCaptionSchema,
+            TaskTypeEnum.DEDUP: IngestTaskDedupSchema,
+            TaskTypeEnum.EMBED: IngestTaskEmbedSchema,
+            TaskTypeEnum.EXTRACT: IngestTaskExtractSchema,
+            TaskTypeEnum.FILTER: IngestTaskFilterSchema,
+            TaskTypeEnum.SPLIT: IngestTaskSplitSchema,
+            TaskTypeEnum.STORE_EMBEDDING: IngestTaskStoreEmbedSchema,
+            TaskTypeEnum.STORE: IngestTaskStoreSchema,
+            TaskTypeEnum.VDB_UPLOAD: IngestTaskVdbUploadSchema,
+            TaskTypeEnum.AUDIO_DATA_EXTRACT: IngestTaskAudioExtraction,
+            TaskTypeEnum.TABLE_DATA_EXTRACT: IngestTaskTableExtraction,
+            TaskTypeEnum.CHART_DATA_EXTRACT: IngestTaskChartExtraction,
+            TaskTypeEnum.INFOGRAPHIC_DATA_EXTRACT: IngestTaskInfographicExtraction,
+        }
+
+        expected_schema_cls = task_type_to_schema.get(task_type)
+        if expected_schema_cls is None:
+            raise ValueError(f"Unsupported or missing task_type '{task_type}'")
+
+        validated_task_properties = expected_schema_cls(**task_properties)
+        values["type"] = task_type  # ensure type is now always the enum
+        values["task_properties"] = validated_task_properties
         return values
 
     @field_validator("type", mode="before")

nv_ingest_api-2025.5.15.dev20250515/src/nv_ingest_api/internal/schemas/transform/transform_text_embedding_schema.py

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+
+import logging
+
+from pydantic import ConfigDict, BaseModel, Field
+
+from nv_ingest_api.util.logging.configuration import LogLevel
+
+logger = logging.getLogger(__name__)
+
+
+class TextEmbeddingSchema(BaseModel):
+    api_key: str = Field(default="api_key")
+    batch_size: int = Field(default=4)
+    embedding_model: str = Field(default="nvidia/nv-embedqa-e5-v5")
+    embedding_nim_endpoint: str = Field(default="http://embedding:8000/v1")
+    encoding_format: str = Field(default="float")
+    httpx_log_level: LogLevel = Field(default=LogLevel.WARNING)
+    input_type: str = Field(default="passage")
+    raise_on_failure: bool = Field(default=False)
+    truncate: str = Field(default="END")
+
+    model_config = ConfigDict(extra="forbid")

{nv_ingest_api-2025.5.13.dev20250513 → nv_ingest_api-2025.5.15.dev20250515}/src/nv_ingest_api/internal/schemas/transform/transform_text_splitter_schema.py

@@ -2,21 +2,23 @@
 # All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-from pydantic import Field, BaseModel, field_validator
+from pydantic import Field, BaseModel, field_validator, ConfigDict
 
 from typing import Optional
 
-from typing_extensions import Annotated
-
 
 class TextSplitterSchema(BaseModel):
     tokenizer: Optional[str] = None
-    chunk_size: Annotated[int, Field(gt=0)] = 1024
-    chunk_overlap: Annotated[int, Field(ge=0)] = 150
+    chunk_size: int = Field(default=1024, gt=0)
+    chunk_overlap: int = Field(default=150, ge=0)
     raise_on_failure: bool = False
 
     @field_validator("chunk_overlap")
-    def check_chunk_overlap(cls, v, values, **kwargs):
-        if v is not None and "chunk_size" in values.data and v >= values.data["chunk_size"]:
+    @classmethod
+    def check_chunk_overlap(cls, v, values):
+        chunk_size = values.data.get("chunk_size")
+        if chunk_size is not None and v >= chunk_size:
             raise ValueError("chunk_overlap must be less than chunk_size")
         return v
+
+    model_config = ConfigDict(extra="forbid")

{nv_ingest_api-2025.5.13.dev20250513 → nv_ingest_api-2025.5.15.dev20250515}/src/nv_ingest_api/internal/store/image_upload.py

@@ -116,6 +116,7 @@ def _upload_images_to_minio(df: pd.DataFrame, params: Dict[str, Any]) -> pd.Data
             if "content" not in metadata:
                 logger.error("Row %s: missing 'content' in metadata", idx)
                 continue
+
             if "source_metadata" not in metadata or not isinstance(metadata["source_metadata"], dict):
                 logger.error("Row %s: missing or invalid 'source_metadata' in metadata", idx)
                 continue

{nv_ingest_api-2025.5.13.dev20250513 → nv_ingest_api-2025.5.15.dev20250515}/src/nv_ingest_api/util/image_processing/processing.py

@@ -150,7 +150,7 @@ def extract_tables_and_charts_yolox(
             min_score=YOLOX_MIN_SCORE,
             final_thresh=YOLOX_FINAL_SCORE,
             trace_info=trace_info,
-            stage_name="pdf_content_extractor",
+            stage_name="pdf_extraction",
         )
 
         # Process results: iterate over each image's inference output.

{nv_ingest_api-2025.5.13.dev20250513 → nv_ingest_api-2025.5.15.dev20250515}/src/nv_ingest_api/util/pdf/pdfium.py

@@ -119,7 +119,7 @@ def pdfium_try_get_bitmap_as_numpy(image_obj) -> np.ndarray:
     return img_array
 
 
-@traceable_func(trace_name="pdf_content_extractor::pdfium_pages_to_numpy")
+@traceable_func(trace_name="pdf_extraction::pdfium_pages_to_numpy")
 def pdfium_pages_to_numpy(
     pages: List[pdfium.PdfPage],
     render_dpi: int = 300,

nv_ingest_api-2025.5.15.dev20250515/src/nv_ingest_api/util/schema/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0

nv_ingest_api-2025.5.15.dev20250515/src/nv_ingest_api/util/service_clients/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0

nv_ingest_api-2025.5.15.dev20250515/src/nv_ingest_api/util/service_clients/redis/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0

{nv_ingest_api-2025.5.13.dev20250513 → nv_ingest_api-2025.5.15.dev20250515}/src/nv_ingest_api/util/system/hardware_info.py

@@ -45,6 +45,10 @@ class SystemResourceProbe:
             A value of 0.5 suggests a hyperthread adds 50% extra performance.
             Requires psutil to be installed and report physical cores.
             Defaults to 0.75.
+
+            Note: the default value of 0.75 is a heuristic and may not be optimal
+            for all situations. It is where parallel pdf decomposition efficiency
+            is observed to begin rolling off.
         """
         if not (0.0 <= hyperthread_weight <= 1.0):
             raise ValueError("hyperthread_weight must be between 0.0 and 1.0")

{nv_ingest_api-2025.5.13.dev20250513 → nv_ingest_api-2025.5.15.dev20250515/src/nv_ingest_api.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-api
-Version: 2025.5.13.dev20250513
+Version: 2025.5.15.dev20250515
 Summary: Python module with core document ingestion functions.
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

nv_ingest_api-2025.5.13.dev20250513/src/nv_ingest_api/internal/schemas/message_brokers/message_broker_client_schema.py

@@ -1,23 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
-# All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-
-from typing import Optional, Literal
-
-from pydantic import Field, BaseModel
-from typing_extensions import Annotated
-
-
-class MessageBrokerClientSchema(BaseModel):
-    host: str = "redis"
-    port: Annotated[int, Field(gt=0, lt=65536)] = 6379
-
-    # Update this for new broker types
-    client_type: Literal["redis", "simple"] = "redis"  # Restrict to 'redis' or 'simple'
-
-    broker_params: Optional[dict] = Field(default_factory=dict)
-
-    connection_timeout: Optional[Annotated[int, Field(ge=0)]] = 300
-    max_backoff: Optional[Annotated[int, Field(ge=0)]] = 300
-    max_retries: Optional[Annotated[int, Field(ge=0)]] = 0

nv_ingest_api-2025.5.13.dev20250513/src/nv_ingest_api/internal/schemas/transform/transform_text_embedding_schema.py

@@ -1,25 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
-# All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-
-import logging
-
-from pydantic import ConfigDict, BaseModel
-
-from nv_ingest_api.util.logging.configuration import LogLevel
-
-logger = logging.getLogger(__name__)
-
-
-class TextEmbeddingSchema(BaseModel):
-    api_key: str = "api_key"
-    batch_size: int = 4
-    embedding_model: str = "nvidia/nv-embedqa-e5-v5"
-    embedding_nim_endpoint: str = "http://embedding:8000/v1"
-    encoding_format: str = "float"
-    httpx_log_level: LogLevel = LogLevel.WARNING
-    input_type: str = "passage"
-    raise_on_failure: bool = False
-    truncate: str = "END"
-    model_config = ConfigDict(extra="forbid")