nv-ingest 2025.10.27.dev20251027__tar.gz → 2025.10.29.dev20251029__tar.gz

{nv_ingest-2025.10.27.dev20251027 → nv_ingest-2025.10.29.dev20251029}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.10.27.dev20251027
+Version: 2025.10.29.dev20251029
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest-2025.10.27.dev20251027 → nv_ingest-2025.10.29.dev20251029}/nv_ingest/api/v2/README.md

@@ -11,15 +11,6 @@ The V2 API introduces automatic PDF splitting at the REST layer to improve proce
 3. **Transparent Aggregation**: Results are automatically aggregated when fetching parent jobs
 4. **Backward Compatible**: PDFs with page counts ≤ `PDF_SPLIT_PAGE_COUNT` behave identical to V1
 
-## Tracing & Aggregated Metadata
-
-- V2 endpoints open an OpenTelemetry span using the shared `traced_endpoint` decorator. The span name defaults to the function name, or can be overridden when applying the decorator.
-- `submit_job_v2` records the parent span's `trace_id` into each subjob's `tracing_options`, enabling downstream Ray stages (e.g., the message broker sink) to attach chunk-level telemetry consistently.
-- Response headers still return `x-trace-id` derived from the active span context, allowing clients to correlate downstream work.
-- When `/v2/fetch_job/{parent_id}` aggregates completed chunks, it captures any `trace` / `annotations` dictionaries emitted by the sink for each subjob and includes them in the response payload (see "Aggregated response" below).
-
-This behaviour matches the V1 tracing model and sets the foundation for adding W3C `traceparent` propagation in future changes.
-
 ## How It Works
 
 1. **Submit**: When a PDF with pages exceeding `PDF_SPLIT_PAGE_COUNT` is submitted to `/v2/submit_job`:
@@ -36,6 +27,33 @@ This behaviour matches the V1 tracing model and sets the foundation for adding W
    - Pending work returns 202 (processing)
    - Failed chunks are noted without failing the entire job; metadata records which chunks failed
 
+
+## Client Library Features
+
+### Accessing Trace Metrics
+
+The Python client library provides convenient access to trace metrics via the `return_traces` parameter:
+
+```python
+from nv_ingest_client.client import Ingestor
+
+ingestor = Ingestor(
+    message_client_hostname="localhost",
+    message_client_port=7670,
+    message_client_kwargs={"api_version": "v2"}
+).files("/path/to/pdfs").extract().embed()
+
+# Get results with trace metrics
+results, traces = ingestor.ingest(return_traces=True)
+
+# Access timing for first document
+pdf_time = traces[0]["trace::resident_time::pdf_extractor"] / 1e9
+table_time = traces[0]["trace::resident_time::table_extractor"] / 1e9
+print(f"PDF: {pdf_time:.2f}s, Tables: {table_time:.2f}s")
+```
+
+**Note:** For split PDFs, `resident_time` represents aggregated compute time across all chunks. For non-split PDFs, it is computed client-side from entry/exit pairs.
+
 ### Aggregated response
 
 The fetch endpoint returns a JSON body shaped like the following:
@@ -163,15 +181,23 @@ For split PDFs, parent-level metrics are automatically computed for each stage (
 - Failed chunk entries remain in `failed_subjobs`; missing chunks indicate the sink did not emit telemetry
 - **To access chunk traces:** Use `metadata.trace_segments[]` - each segment contains the full trace dict for that chunk
 
-## Testing
+### Advanced: Accessing Full Metadata
 
-Use the V2 test script with environment variable:
-```bash
-# Run with V2 endpoints
-DATASET_DIR=/data/splits python scripts/tests/cases/dc20_v2_e2e.py
-```
+For advanced use cases requiring per-chunk trace breakdown or full metadata, use `include_parent_trace_ids`:
+
+```python
+results, traces, parent_trace_ids = ingestor.ingest(
+    return_traces=True,
+    include_parent_trace_ids=True
+)
+
+# Fetch full parent job metadata (including trace_segments)
+import requests
+response = requests.get(f"http://localhost:7670/v2/fetch_job/{parent_trace_ids[0]}")
+metadata = response.json()["metadata"]
 
-Or set the API version for any existing code:
-```bash
-export NV_INGEST_API_VERSION=v2
+# Access per-chunk traces
+for segment in metadata["trace_segments"]:
+    print(f"Chunk {segment['chunk_index']}: pages {segment['start_page']}-{segment['end_page']}")
+    print(f"  Traces: {len(segment['trace'])} entries")
 ```

{nv_ingest-2025.10.27.dev20251027 → nv_ingest-2025.10.29.dev20251029}/nv_ingest/api/v2/ingest.py

@@ -432,6 +432,76 @@ def _extract_ray_telemetry(result: Dict[str, Any]) -> Tuple[Optional[Dict[str, A
     return trace_dict, annotations_dict
 
 
+def _normalize_chunk_records(
+    records: Optional[List[Any]],
+    descriptor: Dict[str, Any],
+    parent_metadata: Dict[str, Any],
+) -> List[Any]:
+    """Re-map chunk-local metadata to document-level context for aggregation."""
+
+    if not isinstance(records, list):
+        return []
+
+    total_pages = parent_metadata.get("total_pages")
+    original_source_id = parent_metadata.get("original_source_id")
+    original_source_name = parent_metadata.get("original_source_name")
+
+    start_page = descriptor.get("start_page")
+    page_offset = start_page - 1 if isinstance(start_page, int) and start_page > 0 else 0
+
+    normalized_entries: List[Any] = []
+
+    for entry in records:
+        if not isinstance(entry, dict):
+            normalized_entries.append(entry)
+            continue
+
+        normalized_entry = entry.copy()
+        original_metadata = entry.get("metadata")
+
+        if isinstance(original_metadata, dict):
+            normalized_metadata = original_metadata.copy()
+            normalized_entry["metadata"] = normalized_metadata
+
+            original_source_meta = original_metadata.get("source_metadata")
+            if isinstance(original_source_meta, dict):
+                normalized_source_meta = original_source_meta.copy()
+                normalized_metadata["source_metadata"] = normalized_source_meta
+
+                if original_source_id:
+                    normalized_source_meta["source_id"] = original_source_id
+                if original_source_name:
+                    normalized_source_meta["source_name"] = original_source_name
+
+            original_content_meta = original_metadata.get("content_metadata")
+            if isinstance(original_content_meta, dict):
+                normalized_content_meta = original_content_meta.copy()
+                normalized_metadata["content_metadata"] = normalized_content_meta
+
+                page_number = normalized_content_meta.get("page_number")
+                if isinstance(page_number, int) and page_number >= 0:
+                    normalized_content_meta["page_number"] = page_number + page_offset
+
+                if isinstance(total_pages, int) and isinstance(normalized_content_meta.get("page_count"), int):
+                    # Ensure optional per-record page count reflects the full document
+                    normalized_content_meta["page_count"] = total_pages
+
+                original_hierarchy = original_content_meta.get("hierarchy")
+                if isinstance(original_hierarchy, dict):
+                    normalized_hierarchy = original_hierarchy.copy()
+                    normalized_content_meta["hierarchy"] = normalized_hierarchy
+
+                    hierarchy_page = normalized_hierarchy.get("page")
+                    if isinstance(hierarchy_page, int) and hierarchy_page >= 0:
+                        normalized_hierarchy["page"] = hierarchy_page + page_offset
+                    if isinstance(total_pages, int):
+                        normalized_hierarchy["page_count"] = total_pages
+
+        normalized_entries.append(normalized_entry)
+
+    return normalized_entries
+
+
 def _aggregate_parent_traces(chunk_traces: Dict[str, Any]) -> Dict[str, Any]:
     """
     Aggregate chunk-level traces into parent-level metrics.
@@ -574,7 +644,8 @@ def _build_aggregated_response(
         if result is not None:
             # Add page data to aggregated result
             if "data" in result:
-                aggregated_result["data"].extend(result["data"])
+                normalized_records = _normalize_chunk_records(result.get("data"), descriptor, metadata)
+                aggregated_result["data"].extend(normalized_records)
             chunk_entry = dict(descriptor)
             aggregated_result["metadata"]["chunks"].append(chunk_entry)
 

{nv_ingest-2025.10.27.dev20251027 → nv_ingest-2025.10.29.dev20251029}/nv_ingest/framework/orchestration/process/dependent_services.py

@@ -18,6 +18,18 @@ from nv_ingest_api.util.message_brokers.simple_message_broker.broker import Simp
 logger = logging.getLogger(__name__)
 
 
+def _broker_server_target(host, port, max_queue_size):
+    """
+    Target function to be run in a separate process for the SimpleMessageBroker.
+    """
+    server = SimpleMessageBroker(host, port, max_queue_size)
+    try:
+        server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    except Exception:
+        pass
+    server.serve_forever()
+
+
 def start_simple_message_broker(broker_client: dict) -> multiprocessing.Process:
     """
     Starts a SimpleMessageBroker server in a separate process.
@@ -58,16 +70,11 @@ def start_simple_message_broker(broker_client: dict) -> multiprocessing.Process:
             f"continuing to spawn a broker process (tests expect a Process to be returned)"
         )
 
-    def broker_server():
-        # Optionally, set socket options here for reuse (note: binding occurs in server __init__).
-        server = SimpleMessageBroker(server_host, server_port, max_queue_size)
-        try:
-            server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        except Exception:
-            pass
-        server.serve_forever()
-
-    p = multiprocessing.Process(target=broker_server)
+    p = multiprocessing.Process(
+        target=_broker_server_target,
+        args=(server_host, server_port, max_queue_size),
+        daemon=True,
+    )
     # If we're launching from inside the pipeline subprocess, mark daemon so the
     # broker dies automatically when the subprocess exits.
     p.daemon = os.environ.get("NV_INGEST_BROKER_IN_SUBPROCESS") == "1"

{nv_ingest-2025.10.27.dev20251027 → nv_ingest-2025.10.29.dev20251029}/nv_ingest/framework/orchestration/process/strategies.py

@@ -11,9 +11,10 @@ Strategy pattern for clean separation of execution concerns.
 """
 
 import atexit
-import os
 import logging
 import multiprocessing
+import os
+import sys
 import time
 from abc import ABC, abstractmethod
 
@@ -132,7 +133,10 @@ class SubprocessStrategy(ProcessExecutionStrategy):
         logger.info("Launching pipeline in Python subprocess using multiprocessing.")
 
         # Create subprocess using fork context
-        ctx = multiprocessing.get_context("fork")
+        start_method = "fork"
+        if sys.platform.lower() == "darwin":
+            start_method = "spawn"
+        ctx = multiprocessing.get_context(start_method)
         process = ctx.Process(
             target=run_pipeline_process,
             args=(

{nv_ingest-2025.10.27.dev20251027 → nv_ingest-2025.10.29.dev20251029}/nv_ingest/framework/util/service/impl/ingest/redis_ingest_service.py

@@ -501,21 +501,21 @@ class RedisIngestService(IngestServiceMeta):
         metadata_key = f"parent:{parent_job_id}:metadata"
 
         try:
-            # Check if this is a parent job
+            # Check if this is a parent job (check metadata_key since non-split PDFs may not have parent_key)
             exists = await self._run_bounded_to_thread(
                 self._ingest_client.get_client().exists,
-                parent_key,
+                metadata_key,  # Check metadata instead of parent_key for non-split PDF support
             )
 
             if not exists:
                 return None
 
-            # Get subjob IDs
+            # Get subjob IDs (may be empty for non-split PDFs)
             subjob_ids_bytes = await self._run_bounded_to_thread(
                 self._ingest_client.get_client().smembers,
                 parent_key,
             )
-            subjob_id_set = {id.decode("utf-8") for id in subjob_ids_bytes}
+            subjob_id_set = {id.decode("utf-8") for id in subjob_ids_bytes} if subjob_ids_bytes else set()
 
             # Get metadata
             metadata_dict = await self._run_bounded_to_thread(

{nv_ingest-2025.10.27.dev20251027 → nv_ingest-2025.10.29.dev20251029}/nv_ingest.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.10.27.dev20251027
+Version: 2025.10.29.dev20251029
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License