nv-ingest 2025.8.8.dev20250808__tar.gz → 2026.1.12.dev20260112__tar.gz

{nv_ingest-2025.8.8.dev20250808 → nv_ingest-2026.1.12.dev20260112}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.8.8.dev20250808
+Version: 2026.1.12.dev20260112
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License
@@ -219,16 +219,19 @@ Requires-Dist: diskcache>=5.6.3
 Requires-Dist: fastapi>=0.115.6
 Requires-Dist: fastparquet>=2024.11.0
 Requires-Dist: fsspec>=2024.10.0
+Requires-Dist: universal_pathlib>=0.2.6
+Requires-Dist: s3fs>=2024.10.0
 Requires-Dist: gunicorn
 Requires-Dist: h11>=0.16.0
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: isodate>=0.7.2
 Requires-Dist: langdetect>=1.0.9
 Requires-Dist: minio>=7.2.12
-Requires-Dist: openai>=1.82.0
+Requires-Dist: librosa==0.10.2
 Requires-Dist: opentelemetry-api>=1.27.0
 Requires-Dist: opentelemetry-exporter-otlp>=1.27.0
 Requires-Dist: opentelemetry-sdk>=1.27.0
+Requires-Dist: psutil>=7.1.0
 Requires-Dist: pydantic>2.0.0
 Requires-Dist: pydantic-settings>2.0.0
 Requires-Dist: pypdfium2==4.30.0
@@ -240,7 +243,7 @@ Requires-Dist: python-docx>=1.1.2
 Requires-Dist: python-dotenv>=1.0.1
 Requires-Dist: python-pptx>=1.0.2
 Requires-Dist: prometheus-client
-Requires-Dist: ray[all]>=2.37.0
+Requires-Dist: ray[all]>=2.49.0
 Requires-Dist: redis>=5.2.1
 Requires-Dist: requests>=2.28.2
 Requires-Dist: scikit-learn>=1.6.0

nv_ingest-2026.1.12.dev20260112/nv_ingest/api/__init__.py

@@ -0,0 +1,9 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""nv_ingest.api package."""
+
+from .tracing import traced_endpoint  # re-export for convenience
+
+__all__ = ["traced_endpoint"]

{nv_ingest-2025.8.8.dev20250808 → nv_ingest-2026.1.12.dev20260112}/nv_ingest/api/main.py

@@ -15,6 +15,7 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor
 from .v1.health import router as HealthApiRouter
 from .v1.ingest import router as IngestApiRouter
 from .v1.metrics import router as MetricsApiRouter
+from .v2.ingest import router as IngestApiRouterV2
 
 logger = logging.getLogger(__name__)
 
@@ -22,7 +23,7 @@ logger = logging.getLogger(__name__)
 app = FastAPI(
     title="NV-Ingest Microservice",
     description="Service for ingesting heterogenous datatypes",
-    version="25.4.2",
+    version="26.1.0",
     contact={
         "name": "NVIDIA Corporation",
         "url": "https://nvidia.com",
@@ -33,6 +34,7 @@ app = FastAPI(
 app.include_router(IngestApiRouter, prefix="/v1")
 app.include_router(HealthApiRouter, prefix="/v1/health")
 app.include_router(MetricsApiRouter, prefix="/v1")
+app.include_router(IngestApiRouterV2, prefix="/v2")
 
 # Set up the tracer provider and add a processor for exporting traces
 resource = Resource(attributes={"service.name": "nv-ingest"})

nv_ingest-2026.1.12.dev20260112/nv_ingest/api/tracing.py

@@ -0,0 +1,82 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""HTTP endpoint tracing utilities."""
+
+from functools import wraps
+from inspect import iscoroutinefunction
+from typing import Any, Callable, Optional, TypeVar
+
+from fastapi import Request, Response
+from opentelemetry import trace
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+tracer = trace.get_tracer(__name__)
+
+
+def traced_endpoint(name: Optional[str] = None) -> Callable[[F], F]:
+    """Wrap a FastAPI endpoint with a span whose name defaults to the function name.
+
+    The decorator preserves the wrapped callable's signature so FastAPI can continue
+    to perform dependency injection and generate OpenAPI documentation correctly.
+    """
+
+    def decorator(func: F) -> F:
+        span_name = name or func.__name__
+
+        if iscoroutinefunction(func):
+
+            @wraps(func)
+            async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
+                with tracer.start_as_current_span(span_name) as span:
+                    span.set_attribute("nv_ingest.endpoint", func.__qualname__)
+                    _record_http_request(span, args, kwargs)
+                    response = await func(*args, **kwargs)
+                    _record_http_response(span, response)
+                    return response
+
+            return async_wrapper  # type: ignore[return-value]
+
+        @wraps(func)
+        def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
+            with tracer.start_as_current_span(span_name) as span:
+                span.set_attribute("nv_ingest.endpoint", func.__qualname__)
+                _record_http_request(span, args, kwargs)
+                result = func(*args, **kwargs)
+                _record_http_response(span, result)
+                return result
+
+        return sync_wrapper  # type: ignore[return-value]
+
+    return decorator
+
+
+def _record_http_request(span, args: tuple[Any, ...], kwargs: dict[str, Any]) -> None:
+    request = _find_type(Request, args, kwargs)
+    if request is None:
+        return
+    span.set_attribute("http.method", request.method)
+    span.set_attribute("http.url", str(request.url))
+
+
+def _record_http_response(span, response: Any) -> None:
+    maybe_response = response if isinstance(response, Response) else None
+    if maybe_response is None:
+        maybe_response = _find_type(Response, (response,), {})
+    if maybe_response is None:
+        return
+    span.set_attribute("http.status_code", maybe_response.status_code)
+
+
+def _find_type(expected_type: type, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Optional[Any]:
+    """Return the first argument matching ``expected_type`` from args or kwargs."""
+
+    for arg in args:
+        if isinstance(arg, expected_type):
+            return arg
+    for value in kwargs.values():
+        if isinstance(value, expected_type):
+            return value
+    return None

nv_ingest-2026.1.12.dev20260112/nv_ingest/api/v2/README.md

@@ -0,0 +1,203 @@
+# NV-Ingest V2 API
+
+## Overview
+
+The V2 API introduces automatic PDF splitting at the REST layer to improve processing throughput. When a multi-page PDF is submitted, it's automatically split into configurable multi-page chunks (default 32 pages) before being sent to the Redis service that then communicates with our Ray processing backend.
+
+## Key Changes from V1
+
+1. **Automatic PDF Splitting**: PDFs over the configured `PDF_SPLIT_PAGE_COUNT` are automatically split into multi-page chunks
+2. **Parent-Child Job Tracking**: Parent jobs maintain relationships with their subjobs via Redis
+3. **Transparent Aggregation**: Results are automatically aggregated when fetching parent jobs
+4. **Backward Compatible**: PDFs with page counts ≤ `PDF_SPLIT_PAGE_COUNT` behave identical to V1
+
+## How It Works
+
+1. **Submit**: When a PDF with pages exceeding `PDF_SPLIT_PAGE_COUNT` is submitted to `/v2/submit_job`:
+   - The PDF is split into page chunks (size determined by `PDF_SPLIT_PAGE_COUNT`)
+   - Each chunk becomes a subjob with deterministic IDs derived from the parent
+   - Source IDs are modified to maintain association: `document.pdf#page_1`
+   - Parent-child mapping is stored in Redis
+
+2. **Processing**: Each subjob is processed independently by Ray, appearing as chunk-sized PDFs that honor the configured `PDF_SPLIT_PAGE_COUNT`
+
+3. **Fetch**: When fetching the parent job via `/v2/fetch_job/{parent_id}`:
+   - Subjob states and results are retrieved concurrently (bounded by the Redis connection pool)
+   - If all complete, results are aggregated in original page order
+   - Pending work returns 202 (processing)
+   - Failed chunks are noted without failing the entire job; metadata records which chunks failed
+
+
+## Client Library Features
+
+### Accessing Trace Metrics
+
+The Python client library provides convenient access to trace metrics via the `return_traces` parameter:
+
+```python
+from nv_ingest_client.client import Ingestor
+
+ingestor = Ingestor(
+    message_client_hostname="localhost",
+    message_client_port=7670,
+    message_client_kwargs={"api_version": "v2"}
+).files("/path/to/pdfs").extract().embed()
+
+# Get results with trace metrics
+results, traces = ingestor.ingest(return_traces=True)
+
+# Access timing for first document
+pdf_time = traces[0]["trace::resident_time::pdf_extractor"] / 1e9
+table_time = traces[0]["trace::resident_time::table_extractor"] / 1e9
+print(f"PDF: {pdf_time:.2f}s, Tables: {table_time:.2f}s")
+```
+
+**Note:** For split PDFs, `resident_time` represents aggregated compute time across all chunks. For non-split PDFs, it is computed client-side from entry/exit pairs.
+
+### Aggregated response
+
+The fetch endpoint returns a JSON body shaped like the following:
+
+```json
+{
+  "data": [...],
+  "status": "success",
+  "trace": {
+    "trace::entry::pdf_extractor": 1000,
+    "trace::exit::pdf_extractor": 2150,
+    "trace::resident_time::pdf_extractor": 250,
+    "trace::entry::table_extractor": 1200,
+    "trace::exit::table_extractor": 2300,
+    "trace::resident_time::table_extractor": 300
+    // ... parent-level aggregated traces only (clean, V1-compatible)
+  },
+  "annotations": {
+    "annotation::uuid-1": {"task_id": "pdf_extractor", "task_result": "SUCCESS"},
+    "annotation::uuid-2": {"task_id": "table_extractor", "task_result": "SUCCESS"}
+    // ... all annotations from all chunks (annotations have unique UUIDs)
+  },
+  "metadata": {
+    "parent_job_id": "<uuid>",
+    "total_pages": 320,
+    "pages_per_chunk": 32,
+    "original_source_id": "document.pdf",
+    "subjob_ids": ["...", "..."],
+    "subjobs_failed": 0,
+    "failed_subjobs": [],
+    "chunks": [
+      {
+        "job_id": "...",
+        "chunk_index": 1,
+        "start_page": 1,
+        "end_page": 32,
+        "page_count": 32
+      }
+      // ... additional chunks ...
+    ],
+    "trace_segments": [
+      {
+        "job_id": "...",
+        "chunk_index": 1,
+        "start_page": 1,
+        "end_page": 32,
+        "trace": {"trace::entry::pdf_extractor": 1.7599e18, ...}
+      }
+      // ... per-chunk trace details
+    ],
+    "annotation_segments": [
+      {
+        "job_id": "...",
+        "chunk_index": 1,
+        "start_page": 1,
+        "end_page": 32,
+        "annotations": {"annotation::uuid": {...}, ...}
+      }
+      // ... per-chunk annotation details
+    ]
+  }
+}
+```
+
+**Top-level trace and annotations** (V1 compatibility):
+- `trace`: Contains **only parent-level aggregated traces** for clean V1 compatibility
+  - `trace::entry::<stage>` - Earliest entry time across all chunks
+  - `trace::exit::<stage>` - Latest exit time across all chunks
+  - `trace::resident_time::<stage>` - Sum of all chunk durations (total compute time)
+- `annotations`: Merged annotations from all chunks (annotations have unique UUIDs so merge safely)
+- These fields match V1 structure, allowing existing client code to work without modification
+
+**Note:** Chunk-level trace details are available in `metadata.trace_segments[]` for granular analysis
+
+**Parent-Level Trace Aggregation:**
+
+For split PDFs, parent-level metrics are automatically computed for each stage (including nested stages):
+
+- `trace::entry::<stage>` - Earliest entry time across all chunks (when first chunk entered stage)
+- `trace::exit::<stage>` - Latest exit time across all chunks (when last chunk exited stage)
+- `trace::resident_time::<stage>` - Sum of all chunk durations (total compute time in stage)
+
+**Supports arbitrary nesting depth:**
+- Simple: `trace::entry::pdf_extractor`
+- Nested: `trace::entry::pdf_extractor::pdf_extraction::pdfium_pages_to_numpy_0`
+
+**Example:**
+```json
+{
+  "trace": {
+    "trace::entry::pdf_extractor": 1000,
+    "trace::exit::pdf_extractor": 2150,
+    "trace::resident_time::pdf_extractor": 250
+    // ... only parent-level aggregations (clean, concise)
+  },
+  "metadata": {
+    "trace_segments": [
+      {
+        "chunk_index": 1,
+        "start_page": 1,
+        "end_page": 32,
+        "trace": {
+          "trace::entry::pdf_extractor": 1000,
+          "trace::exit::pdf_extractor": 1100
+        }
+      },
+      {
+        "chunk_index": 2,
+        "trace": {
+          "trace::entry::pdf_extractor": 2000,
+          "trace::exit::pdf_extractor": 2150
+        }
+      }
+    ]
+  }
+}
+```
+
+**Note:** `resident_time` represents total compute time (sum of chunk durations), while `exit - entry` shows wall-clock span.
+
+**Detailed metadata** (V2-specific):
+- `trace_segments`: **Chunk-level trace data** with page ranges for granular per-chunk analysis
+- `annotation_segments`: Per-chunk annotation data with page ranges
+- Clients can correlate chunk data by matching `job_id` or `chunk_index` across arrays
+- Failed chunk entries remain in `failed_subjobs`; missing chunks indicate the sink did not emit telemetry
+- **To access chunk traces:** Use `metadata.trace_segments[]` - each segment contains the full trace dict for that chunk
+
+### Advanced: Accessing Full Metadata
+
+For advanced use cases requiring per-chunk trace breakdown or full metadata, use `include_parent_trace_ids`:
+
+```python
+results, traces, parent_trace_ids = ingestor.ingest(
+    return_traces=True,
+    include_parent_trace_ids=True
+)
+
+# Fetch full parent job metadata (including trace_segments)
+import requests
+response = requests.get(f"http://localhost:7670/v2/fetch_job/{parent_trace_ids[0]}")
+metadata = response.json()["metadata"]
+
+# Access per-chunk traces
+for segment in metadata["trace_segments"]:
+    print(f"Chunk {segment['chunk_index']}: pages {segment['start_page']}-{segment['end_page']}")
+    print(f"  Traces: {len(segment['trace'])} entries")
+```