nv-ingest 2025.10.8.dev20251008__tar.gz → 2025.10.10.dev20251010__tar.gz

{nv_ingest-2025.10.8.dev20251008 → nv_ingest-2025.10.10.dev20251010}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.10.8.dev20251008
+Version: 2025.10.10.dev20251010
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License
@@ -230,6 +230,7 @@ Requires-Dist: openai>=1.82.0
 Requires-Dist: opentelemetry-api>=1.27.0
 Requires-Dist: opentelemetry-exporter-otlp>=1.27.0
 Requires-Dist: opentelemetry-sdk>=1.27.0
+Requires-Dist: psutil>=7.1.0
 Requires-Dist: pydantic>2.0.0
 Requires-Dist: pydantic-settings>2.0.0
 Requires-Dist: pypdfium2==4.30.0

nv_ingest-2025.10.10.dev20251010/nv_ingest/api/__init__.py

@@ -0,0 +1,9 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""nv_ingest.api package."""
+
+from .tracing import traced_endpoint  # re-export for convenience
+
+__all__ = ["traced_endpoint"]

{nv_ingest-2025.10.8.dev20251008 → nv_ingest-2025.10.10.dev20251010}/nv_ingest/api/main.py

@@ -15,6 +15,7 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor
 from .v1.health import router as HealthApiRouter
 from .v1.ingest import router as IngestApiRouter
 from .v1.metrics import router as MetricsApiRouter
+from .v2.ingest import router as IngestApiRouterV2
 
 logger = logging.getLogger(__name__)
 
@@ -33,6 +34,7 @@ app = FastAPI(
 app.include_router(IngestApiRouter, prefix="/v1")
 app.include_router(HealthApiRouter, prefix="/v1/health")
 app.include_router(MetricsApiRouter, prefix="/v1")
+app.include_router(IngestApiRouterV2, prefix="/v2")
 
 # Set up the tracer provider and add a processor for exporting traces
 resource = Resource(attributes={"service.name": "nv-ingest"})

nv_ingest-2025.10.10.dev20251010/nv_ingest/api/tracing.py

@@ -0,0 +1,82 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""HTTP endpoint tracing utilities."""
+
+from functools import wraps
+from inspect import iscoroutinefunction
+from typing import Any, Callable, Optional, TypeVar
+
+from fastapi import Request, Response
+from opentelemetry import trace
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+tracer = trace.get_tracer(__name__)
+
+
+def traced_endpoint(name: Optional[str] = None) -> Callable[[F], F]:
+    """Wrap a FastAPI endpoint with a span whose name defaults to the function name.
+
+    The decorator preserves the wrapped callable's signature so FastAPI can continue
+    to perform dependency injection and generate OpenAPI documentation correctly.
+    """
+
+    def decorator(func: F) -> F:
+        span_name = name or func.__name__
+
+        if iscoroutinefunction(func):
+
+            @wraps(func)
+            async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
+                with tracer.start_as_current_span(span_name) as span:
+                    span.set_attribute("nv_ingest.endpoint", func.__qualname__)
+                    _record_http_request(span, args, kwargs)
+                    response = await func(*args, **kwargs)
+                    _record_http_response(span, response)
+                    return response
+
+            return async_wrapper  # type: ignore[return-value]
+
+        @wraps(func)
+        def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
+            with tracer.start_as_current_span(span_name) as span:
+                span.set_attribute("nv_ingest.endpoint", func.__qualname__)
+                _record_http_request(span, args, kwargs)
+                result = func(*args, **kwargs)
+                _record_http_response(span, result)
+                return result
+
+        return sync_wrapper  # type: ignore[return-value]
+
+    return decorator
+
+
+def _record_http_request(span, args: tuple[Any, ...], kwargs: dict[str, Any]) -> None:
+    request = _find_type(Request, args, kwargs)
+    if request is None:
+        return
+    span.set_attribute("http.method", request.method)
+    span.set_attribute("http.url", str(request.url))
+
+
+def _record_http_response(span, response: Any) -> None:
+    maybe_response = response if isinstance(response, Response) else None
+    if maybe_response is None:
+        maybe_response = _find_type(Response, (response,), {})
+    if maybe_response is None:
+        return
+    span.set_attribute("http.status_code", maybe_response.status_code)
+
+
+def _find_type(expected_type: type, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Optional[Any]:
+    """Return the first argument matching ``expected_type`` from args or kwargs."""
+
+    for arg in args:
+        if isinstance(arg, expected_type):
+            return arg
+    for value in kwargs.values():
+        if isinstance(value, expected_type):
+            return value
+    return None

nv_ingest-2025.10.10.dev20251010/nv_ingest/api/v2/README.md

@@ -0,0 +1,104 @@
+# NV-Ingest V2 API
+
+## Overview
+
+The V2 API introduces automatic PDF splitting at the REST layer to improve processing throughput. When a multi-page PDF is submitted, it's automatically split into configurable multi-page chunks (default 32 pages) before being sent to the Redis service that then communicates with our Ray processing backend.
+
+## Key Changes from V1
+
+1. **Automatic PDF Splitting**: PDFs over the configured `PDF_SPLIT_PAGE_COUNT` are automatically split into multi-page chunks
+2. **Parent-Child Job Tracking**: Parent jobs maintain relationships with their subjobs via Redis
+3. **Transparent Aggregation**: Results are automatically aggregated when fetching parent jobs
+4. **Backward Compatible**: PDFs with page counts ≤ `PDF_SPLIT_PAGE_COUNT` behave identical to V1
+
+## Tracing & Aggregated Metadata
+
+- V2 endpoints open an OpenTelemetry span using the shared `traced_endpoint` decorator. The span name defaults to the function name, or can be overridden when applying the decorator.
+- `submit_job_v2` records the parent span's `trace_id` into each subjob's `tracing_options`, enabling downstream Ray stages (e.g., the message broker sink) to attach chunk-level telemetry consistently.
+- Response headers still return `x-trace-id` derived from the active span context, allowing clients to correlate downstream work.
+- When `/v2/fetch_job/{parent_id}` aggregates completed chunks, it captures any `trace` / `annotations` dictionaries emitted by the sink for each subjob and includes them in the response payload (see "Aggregated response" below).
+
+This behaviour matches the V1 tracing model and sets the foundation for adding W3C `traceparent` propagation in future changes.
+
+## How It Works
+
+1. **Submit**: When a PDF with pages exceeding `PDF_SPLIT_PAGE_COUNT` is submitted to `/v2/submit_job`:
+   - The PDF is split into page chunks (size determined by `PDF_SPLIT_PAGE_COUNT`)
+   - Each chunk becomes a subjob with deterministic IDs derived from the parent
+   - Source IDs are modified to maintain association: `document.pdf#page_1`
+   - Parent-child mapping is stored in Redis
+
+2. **Processing**: Each subjob is processed independently by Ray, appearing as chunk-sized PDFs that honor the configured `PDF_SPLIT_PAGE_COUNT`
+
+3. **Fetch**: When fetching the parent job via `/v2/fetch_job/{parent_id}`:
+   - Subjob states and results are retrieved concurrently (bounded by the Redis connection pool)
+   - If all complete, results are aggregated in original page order
+   - Pending work returns 202 (processing)
+   - Failed chunks are noted without failing the entire job; metadata records which chunks failed
+
+### Aggregated response
+
+The fetch endpoint returns a JSON body shaped like the following:
+
+```
+{
+  "data": [...],
+  "status": "success",
+  "metadata": {
+    "parent_job_id": "<uuid>",
+    "total_pages": 320,
+    "pages_per_chunk": 32,
+    "original_source_id": "document.pdf",
+    "subjob_ids": ["...", "..."],
+    "subjobs_failed": 0,
+    "failed_subjobs": [],
+    "chunks": [
+      {
+        "job_id": "...",
+        "chunk_index": 1,
+        "start_page": 1,
+        "end_page": 32,
+        "page_count": 32
+      }
+      // ... additional chunks ...
+    ],
+    "trace_segments": [
+      {
+        "job_id": "...",
+        "chunk_index": 1,
+        "start_page": 1,
+        "end_page": 32,
+        "trace": {"trace::sink_push": 1.7285796e+18, ...}
+      }
+      // ...
+    ],
+    "annotation_segments": [
+      {
+        "job_id": "...",
+        "chunk_index": 1,
+        "start_page": 1,
+        "end_page": 32,
+        "annotations": {"annotation::stage": "sink", ...}
+      }
+      // ...
+    ]
+  }
+}
+```
+
+- `trace_segments` and `annotation_segments` appear only when the sink emits telemetry for a given chunk.
+- Clients can correlate chunk data by matching `job_id` or `chunk_index` across `chunks`, `trace_segments`, and `annotation_segments`.
+- Failed chunk entries remain in `failed_subjobs`; if a chunk is missing from the telemetry arrays, the sink did not emit trace/annotation payloads for that chunk.
+
+## Testing
+
+Use the V2 test script with environment variable:
+```bash
+# Run with V2 endpoints
+DATASET_DIR=/data/splits python scripts/tests/cases/dc20_v2_e2e.py
+```
+
+Or set the API version for any existing code:
+```bash
+export NV_INGEST_API_VERSION=v2
+```