nv-ingest 2025.10.8.dev20251008__py3-none-any.whl → 2025.10.10.dev20251010__py3-none-any.whl

nv_ingest/framework/util/service/impl/ingest/redis_ingest_service.py

@@ -7,9 +7,7 @@ import json
 import logging
 import os
 from json import JSONDecodeError
-from typing import Optional, Dict, Any
-
-from typing import List
+from typing import Optional, Dict, Any, List
 
 import redis
 
@@ -133,6 +131,8 @@ class RedisIngestService(IngestServiceMeta):
         self._bulk_vdb_cache_prefix: str = "vdb_bulk_upload_cache:"
         self._cache_prefix: str = "processing_cache:"
         self._state_prefix: str = "job_state:"
+        # Bound async-to-thread concurrency slightly below Redis connection pool
+        self._async_operation_semaphore: Optional[asyncio.Semaphore] = None
 
         self._ingest_client = RedisClient(
             host=self._redis_hostname,
@@ -151,6 +151,16 @@ class RedisIngestService(IngestServiceMeta):
             f"FetchMode: {fetch_mode.name}, ResultTTL: {result_data_ttl_seconds}, StateTTL: {state_ttl_seconds}"
         )
 
+    def _get_async_semaphore(self) -> asyncio.Semaphore:
+        if self._async_operation_semaphore is None:
+            semaphore_limit = max(1, self._concurrency_level - 2)
+            self._async_operation_semaphore = asyncio.Semaphore(semaphore_limit)
+        return self._async_operation_semaphore
+
+    async def _run_bounded_to_thread(self, func, *args, **kwargs):
+        async with self._get_async_semaphore():
+            return await asyncio.to_thread(func, *args, **kwargs)
+
     async def submit_job(self, job_spec_wrapper: "MessageWrapper", trace_id: str) -> str:
         """
         Validates, prepares, and submits a job specification to the Redis task queue.
@@ -211,7 +221,7 @@ class RedisIngestService(IngestServiceMeta):
             logger.debug(
                 f"Submitting job {trace_id} to queue '{self._redis_task_queue}' with result TTL: {ttl_for_result}"
             )
-            await asyncio.to_thread(
+            await self._run_bounded_to_thread(
                 self._ingest_client.submit_message,
                 channel_name=self._redis_task_queue,
                 message=job_spec_json,
@@ -252,7 +262,7 @@ class RedisIngestService(IngestServiceMeta):
         try:
             result_channel: str = f"{job_id}"
             logger.debug(f"Attempting to fetch job result for {job_id} using mode {self._fetch_mode.name}")
-            message = await asyncio.to_thread(
+            message = await self._run_bounded_to_thread(
                 self._ingest_client.fetch_message,
                 channel_name=result_channel,
                 timeout=10,
@@ -264,7 +274,7 @@ class RedisIngestService(IngestServiceMeta):
                 logger.warning(f"fetch_message for {job_id} returned None unexpectedly.")
                 raise TimeoutError("No data found (unexpected None response).")
         except (TimeoutError, redis.RedisError, ConnectionError, ValueError, RuntimeError) as e:
-            logger.info(f"Fetch operation for job {job_id} did not complete: ({type(e).__name__}) {e}")
+            logger.debug(f"Fetch operation for job {job_id} did not complete: ({type(e).__name__}) {e}")
             raise e
         except Exception as e:
             logger.exception(f"Unexpected error during async fetch_job for {job_id}: {e}")
@@ -289,7 +299,7 @@ class RedisIngestService(IngestServiceMeta):
         ttl_to_set: Optional[int] = self._state_ttl_seconds
         try:
             logger.debug(f"Setting state for {job_id} to {state} with TTL {ttl_to_set}")
-            await asyncio.to_thread(
+            await self._run_bounded_to_thread(
                 self._ingest_client.get_client().set,
                 state_key,
                 state,
@@ -317,7 +327,10 @@ class RedisIngestService(IngestServiceMeta):
         """
         state_key: str = f"{self._state_prefix}{job_id}"
         try:
-            data_bytes: Optional[bytes] = await asyncio.to_thread(self._ingest_client.get_client().get, state_key)
+            data_bytes: Optional[bytes] = await self._run_bounded_to_thread(
+                self._ingest_client.get_client().get,
+                state_key,
+            )
             if data_bytes:
                 state: str = data_bytes.decode("utf-8")
                 logger.debug(f"Retrieved state for {job_id}: {state}")
@@ -350,7 +363,7 @@ class RedisIngestService(IngestServiceMeta):
         cache_key: str = f"{self._cache_prefix}{job_id}"
         try:
             data_to_store: str = json.dumps([job.model_dump(mode="json") for job in jobs_data])
-            await asyncio.to_thread(
+            await self._run_bounded_to_thread(
                 self._ingest_client.get_client().set,
                 cache_key,
                 data_to_store,
@@ -375,7 +388,10 @@ class RedisIngestService(IngestServiceMeta):
         """
         cache_key: str = f"{self._cache_prefix}{job_id}"
         try:
-            data_bytes: Optional[bytes] = await asyncio.to_thread(self._ingest_client.get_client().get, cache_key)
+            data_bytes: Optional[bytes] = await self._run_bounded_to_thread(
+                self._ingest_client.get_client().get,
+                cache_key,
+            )
             if data_bytes is None:
                 return []
             return [ProcessingJob(**job) for job in json.loads(data_bytes)]
@@ -393,3 +409,169 @@ class RedisIngestService(IngestServiceMeta):
             The current fetch mode.
         """
         return self._fetch_mode
+
+    async def set_parent_job_mapping(
+        self,
+        parent_job_id: str,
+        subjob_ids: List[str],
+        metadata: Dict[str, Any],
+        *,
+        subjob_descriptors: Optional[List[Dict[str, Any]]] = None,
+    ) -> None:
+        """
+        Store parent-subjob mapping in Redis for V2 PDF splitting.
+
+        Parameters
+        ----------
+        parent_job_id : str
+            The parent job identifier
+        subjob_ids : List[str]
+            List of subjob identifiers
+        metadata : Dict[str, Any]
+            Metadata about the parent job (total_pages, original_source_id, etc.)
+        subjob_descriptors : List[Dict[str, Any]], optional
+            Detailed descriptors (job_id, chunk_index, start/end pages) for subjobs
+        """
+        parent_key = f"parent:{parent_job_id}:subjobs"
+        metadata_key = f"parent:{parent_job_id}:metadata"
+
+        try:
+            # Store subjob IDs as a set
+            await self._run_bounded_to_thread(
+                self._ingest_client.get_client().sadd,
+                parent_key,
+                *subjob_ids,
+            )
+
+            # Store metadata as hash (including original subjob ordering for deterministic fetches)
+            metadata_to_store = dict(metadata)
+            try:
+                metadata_to_store["subjob_order"] = json.dumps(subjob_ids)
+            except (TypeError, ValueError):
+                logger.warning(
+                    "Unable to serialize subjob ordering for parent %s; falling back to Redis set ordering",
+                    parent_job_id,
+                )
+                metadata_to_store.pop("subjob_order", None)
+
+            if subjob_descriptors:
+                metadata_to_store["subjob_descriptors"] = json.dumps(subjob_descriptors)
+
+            await self._run_bounded_to_thread(
+                self._ingest_client.get_client().hset,
+                metadata_key,
+                mapping=metadata_to_store,
+            )
+
+            # Set TTL on both keys to match state TTL
+            if self._state_ttl_seconds:
+                await self._run_bounded_to_thread(
+                    self._ingest_client.get_client().expire,
+                    parent_key,
+                    self._state_ttl_seconds,
+                )
+                await self._run_bounded_to_thread(
+                    self._ingest_client.get_client().expire,
+                    metadata_key,
+                    self._state_ttl_seconds,
+                )
+
+            logger.debug(f"Stored parent job mapping for {parent_job_id} with {len(subjob_ids)} subjobs")
+
+        except Exception as err:
+            logger.exception(f"Error storing parent job mapping for {parent_job_id}: {err}")
+            raise
+
+    async def get_parent_job_info(self, parent_job_id: str) -> Optional[Dict[str, Any]]:
+        """
+        Retrieve parent job information including subjob IDs and metadata.
+
+        Parameters
+        ----------
+        parent_job_id : str
+            The parent job identifier
+
+        Returns
+        -------
+        Dict[str, Any] or None
+            Dictionary with 'subjob_ids' and 'metadata' keys, or None if not a parent job
+        """
+        parent_key = f"parent:{parent_job_id}:subjobs"
+        metadata_key = f"parent:{parent_job_id}:metadata"
+
+        try:
+            # Check if this is a parent job
+            exists = await self._run_bounded_to_thread(
+                self._ingest_client.get_client().exists,
+                parent_key,
+            )
+
+            if not exists:
+                return None
+
+            # Get subjob IDs
+            subjob_ids_bytes = await self._run_bounded_to_thread(
+                self._ingest_client.get_client().smembers,
+                parent_key,
+            )
+            subjob_id_set = {id.decode("utf-8") for id in subjob_ids_bytes}
+
+            # Get metadata
+            metadata_dict = await self._run_bounded_to_thread(
+                self._ingest_client.get_client().hgetall,
+                metadata_key,
+            )
+            metadata = {k.decode("utf-8"): v.decode("utf-8") for k, v in metadata_dict.items()}
+
+            # Convert numeric strings back to numbers
+            if "total_pages" in metadata:
+                metadata["total_pages"] = int(metadata["total_pages"])
+            if "pages_per_chunk" in metadata:
+                try:
+                    metadata["pages_per_chunk"] = int(metadata["pages_per_chunk"])
+                except ValueError:
+                    metadata.pop("pages_per_chunk", None)
+
+            ordered_ids: Optional[List[str]] = None
+            stored_order = metadata.pop("subjob_order", None)
+            if stored_order:
+                try:
+                    candidate_order = json.loads(stored_order)
+                    if isinstance(candidate_order, list):
+                        ordered_ids = [sid for sid in candidate_order if sid in subjob_id_set]
+                except (ValueError, TypeError) as exc:
+                    logger.warning(
+                        "Failed to parse stored subjob order for parent %s: %s",
+                        parent_job_id,
+                        exc,
+                    )
+
+            if ordered_ids is None:
+                ordered_ids = sorted(subjob_id_set)
+            else:
+                remaining_ids = sorted(subjob_id_set - set(ordered_ids))
+                ordered_ids.extend(remaining_ids)
+
+            subjob_descriptors: Optional[List[Dict[str, Any]]] = None
+            stored_descriptors = metadata.pop("subjob_descriptors", None)
+            if stored_descriptors:
+                try:
+                    decoded = json.loads(stored_descriptors)
+                    if isinstance(decoded, list):
+                        subjob_descriptors = decoded
+                except (ValueError, TypeError) as exc:
+                    logger.warning(
+                        "Failed to parse stored subjob descriptors for parent %s: %s",
+                        parent_job_id,
+                        exc,
+                    )
+
+            return {
+                "subjob_ids": ordered_ids,
+                "metadata": metadata,
+                "subjob_descriptors": subjob_descriptors or [],
+            }
+
+        except Exception as err:
+            logger.error(f"Error retrieving parent job info for {parent_job_id}: {err}")
+            return None

{nv_ingest-2025.10.8.dev20251008.dist-info → nv_ingest-2025.10.10.dev20251010.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.10.8.dev20251008
+Version: 2025.10.10.dev20251010
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License
@@ -230,6 +230,7 @@ Requires-Dist: openai>=1.82.0
 Requires-Dist: opentelemetry-api>=1.27.0
 Requires-Dist: opentelemetry-exporter-otlp>=1.27.0
 Requires-Dist: opentelemetry-sdk>=1.27.0
+Requires-Dist: psutil>=7.1.0
 Requires-Dist: pydantic>2.0.0
 Requires-Dist: pydantic-settings>2.0.0
 Requires-Dist: pypdfium2==4.30.0

{nv_ingest-2025.10.8.dev20251008.dist-info → nv_ingest-2025.10.10.dev20251010.dist-info}/RECORD

@@ -1,11 +1,15 @@
 nv_ingest/__init__.py,sha256=vJLPeuxiIHqbxXPJSu9qe3MS-GPavbOUExyRq83DxxM,895
 nv_ingest/version.py,sha256=MG7DxlzpnoJI56vqxwzs9WeMAEI3uPhfDiNLs6GN6wI,986
-nv_ingest/api/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/api/main.py,sha256=XE-p4lJp1E7CCDOB8ENtYFrf63Dtq2bzQiGxpRfL2LA,1603
+nv_ingest/api/__init__.py,sha256=ED07QUqwVyJalH0ahhnnjvc2W_in6TpZZ5nJ6NWU9-Y,271
+nv_ingest/api/main.py,sha256=uCCkUNLS1xE9TDYKDOdxEfo_9jQWumpQAPWrxj5m9Go,1706
+nv_ingest/api/tracing.py,sha256=NkqMuUiB6ixGU5MYp3TrODsZDQepJ1kbH8JFHsYjuE0,2940
 nv_ingest/api/v1/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/api/v1/health.py,sha256=pV-RoVq5y0iBPp0qZoLzd1xKpd0JiHAi0UMyMj99LqU,4740
 nv_ingest/api/v1/ingest.py,sha256=LWk3LN4lBd3uO8h30EN42g3LHCVcO00avVd5ohVK7NI,19392
 nv_ingest/api/v1/metrics.py,sha256=ZGVRApYLnzc2f2C7wRgGd7deqiXan-jxfA-33a16clY,981
+nv_ingest/api/v2/README.md,sha256=eJHe-AXOczH1FH0qOsQ4PNR1UCkt3nPFcAPcZ6PEDjk,4307
+nv_ingest/api/v2/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
+nv_ingest/api/v2/ingest.py,sha256=XuEMgc1iRNOux83xRuTA5X9drPAR_vGDKEhyHXf9D5Q,32203
 nv_ingest/framework/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/execution/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
@@ -99,7 +103,7 @@ nv_ingest/framework/util/flow_control/udf_intercept.py,sha256=zQ9uuCcHLEd0P52Eiw
 nv_ingest/framework/util/service/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/util/service/impl/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/util/service/impl/ingest/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/framework/util/service/impl/ingest/redis_ingest_service.py,sha256=KbzQFo7qVbCITiKYVPcGN0x4NI8piJy70Dz-8jf59Xs,15415
+nv_ingest/framework/util/service/impl/ingest/redis_ingest_service.py,sha256=OuGC3FFhkLQLR3x4s-tyxGguYYn8ORKr2xkzMy2br0g,22552
 nv_ingest/framework/util/service/meta/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/util/service/meta/ingest/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/util/service/meta/ingest/ingest_service_meta.py,sha256=QS3uNxWBl5dIcmIpJKNe8_TLcTUuN2vcKyHeAwa-eSo,1589
@@ -113,8 +117,8 @@ nv_ingest/pipeline/pipeline_schema.py,sha256=rLZZz2It2o2hVNWrZUJU8CarrqRei1fho3Z
 nv_ingest/pipeline/config/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/pipeline/config/loaders.py,sha256=75Yr9WYO7j7ghvKTnYLfZXQZEH3J3VEZo5J4TunC_Us,7590
 nv_ingest/pipeline/config/replica_resolver.py,sha256=3zjh8gmepEYORFZRM4inq7GoBW0YL3gzUDiixUugjzQ,8899
-nv_ingest-2025.10.8.dev20251008.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-nv_ingest-2025.10.8.dev20251008.dist-info/METADATA,sha256=hhFBvw9s5nCRjXnd5DFOANjkNoUlLLnjJ1M3A9mk2Kk,15092
-nv_ingest-2025.10.8.dev20251008.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nv_ingest-2025.10.8.dev20251008.dist-info/top_level.txt,sha256=sjb0ajIsgn3YgftSjZHlYO0HjYAIIhNuXG_AmywCvaU,10
-nv_ingest-2025.10.8.dev20251008.dist-info/RECORD,,
+nv_ingest-2025.10.10.dev20251010.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+nv_ingest-2025.10.10.dev20251010.dist-info/METADATA,sha256=tflK_t8mmrF2P-mZYhoa673OVzPhqQKOXTFEEJ_77ng,15122
+nv_ingest-2025.10.10.dev20251010.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nv_ingest-2025.10.10.dev20251010.dist-info/top_level.txt,sha256=sjb0ajIsgn3YgftSjZHlYO0HjYAIIhNuXG_AmywCvaU,10
+nv_ingest-2025.10.10.dev20251010.dist-info/RECORD,,