nv-ingest 2025.10.28.dev20251028__tar.gz → 2025.10.29.dev20251029__tar.gz

{nv_ingest-2025.10.28.dev20251028 → nv_ingest-2025.10.29.dev20251029}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.10.28.dev20251028
+Version: 2025.10.29.dev20251029
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest-2025.10.28.dev20251028 → nv_ingest-2025.10.29.dev20251029}/nv_ingest/api/v2/ingest.py

@@ -432,6 +432,76 @@ def _extract_ray_telemetry(result: Dict[str, Any]) -> Tuple[Optional[Dict[str, A
     return trace_dict, annotations_dict
 
 
+def _normalize_chunk_records(
+    records: Optional[List[Any]],
+    descriptor: Dict[str, Any],
+    parent_metadata: Dict[str, Any],
+) -> List[Any]:
+    """Re-map chunk-local metadata to document-level context for aggregation."""
+
+    if not isinstance(records, list):
+        return []
+
+    total_pages = parent_metadata.get("total_pages")
+    original_source_id = parent_metadata.get("original_source_id")
+    original_source_name = parent_metadata.get("original_source_name")
+
+    start_page = descriptor.get("start_page")
+    page_offset = start_page - 1 if isinstance(start_page, int) and start_page > 0 else 0
+
+    normalized_entries: List[Any] = []
+
+    for entry in records:
+        if not isinstance(entry, dict):
+            normalized_entries.append(entry)
+            continue
+
+        normalized_entry = entry.copy()
+        original_metadata = entry.get("metadata")
+
+        if isinstance(original_metadata, dict):
+            normalized_metadata = original_metadata.copy()
+            normalized_entry["metadata"] = normalized_metadata
+
+            original_source_meta = original_metadata.get("source_metadata")
+            if isinstance(original_source_meta, dict):
+                normalized_source_meta = original_source_meta.copy()
+                normalized_metadata["source_metadata"] = normalized_source_meta
+
+                if original_source_id:
+                    normalized_source_meta["source_id"] = original_source_id
+                if original_source_name:
+                    normalized_source_meta["source_name"] = original_source_name
+
+            original_content_meta = original_metadata.get("content_metadata")
+            if isinstance(original_content_meta, dict):
+                normalized_content_meta = original_content_meta.copy()
+                normalized_metadata["content_metadata"] = normalized_content_meta
+
+                page_number = normalized_content_meta.get("page_number")
+                if isinstance(page_number, int) and page_number >= 0:
+                    normalized_content_meta["page_number"] = page_number + page_offset
+
+                if isinstance(total_pages, int) and isinstance(normalized_content_meta.get("page_count"), int):
+                    # Ensure optional per-record page count reflects the full document
+                    normalized_content_meta["page_count"] = total_pages
+
+                original_hierarchy = original_content_meta.get("hierarchy")
+                if isinstance(original_hierarchy, dict):
+                    normalized_hierarchy = original_hierarchy.copy()
+                    normalized_content_meta["hierarchy"] = normalized_hierarchy
+
+                    hierarchy_page = normalized_hierarchy.get("page")
+                    if isinstance(hierarchy_page, int) and hierarchy_page >= 0:
+                        normalized_hierarchy["page"] = hierarchy_page + page_offset
+                    if isinstance(total_pages, int):
+                        normalized_hierarchy["page_count"] = total_pages
+
+        normalized_entries.append(normalized_entry)
+
+    return normalized_entries
+
+
 def _aggregate_parent_traces(chunk_traces: Dict[str, Any]) -> Dict[str, Any]:
     """
     Aggregate chunk-level traces into parent-level metrics.
@@ -574,7 +644,8 @@ def _build_aggregated_response(
         if result is not None:
             # Add page data to aggregated result
             if "data" in result:
-                aggregated_result["data"].extend(result["data"])
+                normalized_records = _normalize_chunk_records(result.get("data"), descriptor, metadata)
+                aggregated_result["data"].extend(normalized_records)
             chunk_entry = dict(descriptor)
             aggregated_result["metadata"]["chunks"].append(chunk_entry)
 

{nv_ingest-2025.10.28.dev20251028 → nv_ingest-2025.10.29.dev20251029}/nv_ingest/framework/orchestration/process/dependent_services.py

@@ -18,6 +18,18 @@ from nv_ingest_api.util.message_brokers.simple_message_broker.broker import Simp
 logger = logging.getLogger(__name__)
 
 
+def _broker_server_target(host, port, max_queue_size):
+    """
+    Target function to be run in a separate process for the SimpleMessageBroker.
+    """
+    server = SimpleMessageBroker(host, port, max_queue_size)
+    try:
+        server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    except Exception:
+        pass
+    server.serve_forever()
+
+
 def start_simple_message_broker(broker_client: dict) -> multiprocessing.Process:
     """
     Starts a SimpleMessageBroker server in a separate process.
@@ -58,16 +70,11 @@ def start_simple_message_broker(broker_client: dict) -> multiprocessing.Process:
             f"continuing to spawn a broker process (tests expect a Process to be returned)"
         )
 
-    def broker_server():
-        # Optionally, set socket options here for reuse (note: binding occurs in server __init__).
-        server = SimpleMessageBroker(server_host, server_port, max_queue_size)
-        try:
-            server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        except Exception:
-            pass
-        server.serve_forever()
-
-    p = multiprocessing.Process(target=broker_server)
+    p = multiprocessing.Process(
+        target=_broker_server_target,
+        args=(server_host, server_port, max_queue_size),
+        daemon=True,
+    )
     # If we're launching from inside the pipeline subprocess, mark daemon so the
     # broker dies automatically when the subprocess exits.
     p.daemon = os.environ.get("NV_INGEST_BROKER_IN_SUBPROCESS") == "1"

{nv_ingest-2025.10.28.dev20251028 → nv_ingest-2025.10.29.dev20251029}/nv_ingest/framework/orchestration/process/strategies.py

@@ -11,9 +11,10 @@ Strategy pattern for clean separation of execution concerns.
 """
 
 import atexit
-import os
 import logging
 import multiprocessing
+import os
+import sys
 import time
 from abc import ABC, abstractmethod
 
@@ -132,7 +133,10 @@ class SubprocessStrategy(ProcessExecutionStrategy):
         logger.info("Launching pipeline in Python subprocess using multiprocessing.")
 
         # Create subprocess using fork context
-        ctx = multiprocessing.get_context("fork")
+        start_method = "fork"
+        if sys.platform.lower() == "darwin":
+            start_method = "spawn"
+        ctx = multiprocessing.get_context(start_method)
         process = ctx.Process(
             target=run_pipeline_process,
             args=(

{nv_ingest-2025.10.28.dev20251028 → nv_ingest-2025.10.29.dev20251029}/nv_ingest/framework/util/service/impl/ingest/redis_ingest_service.py

@@ -501,21 +501,21 @@ class RedisIngestService(IngestServiceMeta):
         metadata_key = f"parent:{parent_job_id}:metadata"
 
         try:
-            # Check if this is a parent job
+            # Check if this is a parent job (check metadata_key since non-split PDFs may not have parent_key)
             exists = await self._run_bounded_to_thread(
                 self._ingest_client.get_client().exists,
-                parent_key,
+                metadata_key,  # Check metadata instead of parent_key for non-split PDF support
             )
 
             if not exists:
                 return None
 
-            # Get subjob IDs
+            # Get subjob IDs (may be empty for non-split PDFs)
             subjob_ids_bytes = await self._run_bounded_to_thread(
                 self._ingest_client.get_client().smembers,
                 parent_key,
             )
-            subjob_id_set = {id.decode("utf-8") for id in subjob_ids_bytes}
+            subjob_id_set = {id.decode("utf-8") for id in subjob_ids_bytes} if subjob_ids_bytes else set()
 
             # Get metadata
             metadata_dict = await self._run_bounded_to_thread(

{nv_ingest-2025.10.28.dev20251028 → nv_ingest-2025.10.29.dev20251029}/nv_ingest.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.10.28.dev20251028
+Version: 2025.10.29.dev20251029
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License