nv-ingest 2025.8.4.dev20250804__py3-none-any.whl → 2025.12.10.dev20251210__py3-none-any.whl

nv_ingest/framework/util/flow_control/udf_intercept.py

@@ -0,0 +1,352 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import functools
+import hashlib
+import inspect
+import logging
+import os
+import time
+from typing import Dict, List, Any, Optional, Callable
+from dataclasses import dataclass
+
+from nv_ingest_api.internal.primitives.ingest_control_message import IngestControlMessage, remove_all_tasks_by_type
+from nv_ingest_api.util.imports.callable_signatures import ingest_callable_signature
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class CachedUDF:
+    """Cached UDF function with metadata"""
+
+    function: callable
+    function_name: str
+    signature_validated: bool
+    created_at: float
+    last_used: float
+    use_count: int
+
+
+class UDFCache:
+    """LRU cache for compiled and validated UDF functions"""
+
+    def __init__(self, max_size: int = 128, ttl_seconds: Optional[int] = 3600):
+        self.max_size = max_size
+        self.ttl_seconds = ttl_seconds
+        self.cache: Dict[str, CachedUDF] = {}
+        self.access_order: List[str] = []  # For LRU tracking
+
+    def _generate_cache_key(self, udf_function_str: str, udf_function_name: str) -> str:
+        """Generate cache key from UDF string and function name"""
+        content = f"{udf_function_str.strip()}:{udf_function_name}"
+        return hashlib.sha256(content.encode()).hexdigest()
+
+    def _evict_lru(self):
+        """Remove least recently used item"""
+        if self.access_order:
+            lru_key = self.access_order.pop(0)
+            self.cache.pop(lru_key, None)
+
+    def _cleanup_expired(self):
+        """Remove expired entries if TTL is configured"""
+        if not self.ttl_seconds:
+            return
+
+        current_time = time.time()
+        expired_keys = [
+            key for key, cached_udf in self.cache.items() if current_time - cached_udf.created_at > self.ttl_seconds
+        ]
+
+        for key in expired_keys:
+            self.cache.pop(key, None)
+            if key in self.access_order:
+                self.access_order.remove(key)
+
+    def get(self, udf_function_str: str, udf_function_name: str) -> Optional[CachedUDF]:
+        """Get cached UDF function if available"""
+        self._cleanup_expired()
+
+        cache_key = self._generate_cache_key(udf_function_str, udf_function_name)
+
+        if cache_key in self.cache:
+            # Update access tracking
+            if cache_key in self.access_order:
+                self.access_order.remove(cache_key)
+            self.access_order.append(cache_key)
+
+            # Update usage stats
+            cached_udf = self.cache[cache_key]
+            cached_udf.last_used = time.time()
+            cached_udf.use_count += 1
+
+            return cached_udf
+
+        return None
+
+    def put(
+        self, udf_function_str: str, udf_function_name: str, function: callable, signature_validated: bool = True
+    ) -> str:
+        """Cache a compiled and validated UDF function"""
+        cache_key = self._generate_cache_key(udf_function_str, udf_function_name)
+
+        # Evict LRU if at capacity
+        while len(self.cache) >= self.max_size:
+            self._evict_lru()
+
+        current_time = time.time()
+        cached_udf = CachedUDF(
+            function=function,
+            function_name=udf_function_name,
+            signature_validated=signature_validated,
+            created_at=current_time,
+            last_used=current_time,
+            use_count=1,
+        )
+
+        self.cache[cache_key] = cached_udf
+        self.access_order.append(cache_key)
+
+        return cache_key
+
+    def get_stats(self) -> Dict[str, Any]:
+        """Get cache statistics"""
+        total_uses = sum(udf.use_count for udf in self.cache.values())
+        most_used = max(self.cache.values(), key=lambda x: x.use_count, default=None)
+        return {
+            "size": len(self.cache),
+            "max_size": self.max_size,
+            "total_uses": total_uses,
+            "most_used_function": most_used.function_name if most_used else None,
+            "most_used_count": most_used.use_count if most_used else 0,
+        }
+
+
+# Global cache instance
+_udf_cache = UDFCache(max_size=128, ttl_seconds=3600)
+
+
+def compile_and_validate_udf(udf_function_str: str, udf_function_name: str, task_num: int) -> callable:
+    """Compile and validate UDF function (extracted for caching)"""
+    # Execute the UDF function string in a controlled namespace
+    namespace: Dict[str, Any] = {}
+    try:
+        exec(udf_function_str, namespace)
+    except Exception as e:
+        raise ValueError(f"UDF task {task_num} failed to execute: {str(e)}")
+
+    # Extract the specified function from the namespace
+    if udf_function_name in namespace and callable(namespace[udf_function_name]):
+        udf_function = namespace[udf_function_name]
+    else:
+        raise ValueError(f"UDF task {task_num}: Specified UDF function '{udf_function_name}' not found or not callable")
+
+    # Validate the UDF function signature
+    try:
+        ingest_callable_signature(inspect.signature(udf_function))
+    except Exception as e:
+        raise ValueError(f"UDF task {task_num} has invalid function signature: {str(e)}")
+
+    return udf_function
+
+
+def execute_targeted_udfs(
+    control_message: IngestControlMessage, stage_name: str, directive: str
+) -> IngestControlMessage:
+    """Execute UDFs that target this stage with the given directive."""
+    # Early exit if no UDF tasks exist - check by task type, not task ID
+    udf_tasks_exist = any(task.type == "udf" for task in control_message.get_tasks())
+    if not udf_tasks_exist:
+        return control_message
+
+    # Remove all UDF tasks and get them - handle case where no tasks found
+    try:
+        all_udf_tasks = remove_all_tasks_by_type(control_message, "udf")
+    except ValueError:
+        # No UDF tasks found - this can happen due to race conditions
+        logger.debug(f"No UDF tasks found for stage '{stage_name}' directive '{directive}'")
+        return control_message
+
+    # Execute applicable UDFs and collect remaining ones
+    remaining_tasks = []
+
+    for task_properties in all_udf_tasks:
+        # Check if this UDF targets this stage with the specified directive
+        target_stage = task_properties.get("target_stage", "")
+        run_before = task_properties.get("run_before", False)
+        run_after = task_properties.get("run_after", False)
+
+        # Determine if this UDF should execute
+        should_execute = False
+        if directive == "run_before" and run_before and target_stage == stage_name:
+            should_execute = True
+        elif directive == "run_after" and run_after and target_stage == stage_name:
+            should_execute = True
+
+        if should_execute:
+            try:
+                # Get UDF function details
+                udf_function_str = task_properties.get("udf_function", "").strip()
+                udf_function_name = task_properties.get("udf_function_name", "").strip()
+                task_id = task_properties.get("task_id", "unknown")
+
+                # Skip empty UDF functions
+                if not udf_function_str:
+                    logger.debug(f"UDF task {task_id} has empty function, skipping")
+                    remaining_tasks.append(task_properties)
+                    continue
+
+                # Validate function name
+                if not udf_function_name:
+                    raise ValueError(f"UDF task {task_id} missing required 'udf_function_name' property")
+
+                # Get or compile UDF function
+                cached_udf = _udf_cache.get(udf_function_str, udf_function_name)
+                if cached_udf:
+                    udf_function = cached_udf.function
+                    logger.debug(f"UDF task {task_id}: Using cached function '{udf_function_name}'")
+                else:
+                    udf_function = compile_and_validate_udf(udf_function_str, udf_function_name, task_id)
+                    _udf_cache.put(udf_function_str, udf_function_name, udf_function)
+                    logger.debug(f"UDF task {task_id}: Cached function '{udf_function_name}'")
+
+                # Execute the UDF
+                control_message = udf_function(control_message)
+
+                # Validate return type
+                if not isinstance(control_message, IngestControlMessage):
+                    raise ValueError(f"UDF task {task_id} must return IngestControlMessage")
+
+                logger.info(f"Executed UDF {task_id} '{udf_function_name}' {directive} stage '{stage_name}'")
+
+            except Exception as e:
+                logger.error(f"UDF {task_id} failed {directive} stage '{stage_name}': {e}")
+                # Keep failed task for next stage
+                remaining_tasks.append(task_properties)
+        else:
+            # Keep non-applicable task for next stage
+            remaining_tasks.append(task_properties)
+
+    # Re-add all remaining UDF tasks
+    for task_properties in remaining_tasks:
+        from nv_ingest_api.internal.primitives.control_message_task import ControlMessageTask
+
+        task = ControlMessageTask(type="udf", id=task_properties.get("task_id", "unknown"), properties=task_properties)
+        control_message.add_task(task)
+
+    return control_message
+
+
+def remove_task_by_id(control_message: IngestControlMessage, task_id: str) -> IngestControlMessage:
+    """Remove a specific task by ID from the control message"""
+    try:
+        control_message.remove_task(task_id)
+    except RuntimeError as e:
+        logger.warning(f"Could not remove task {task_id}: {e}")
+
+    return control_message
+
+
+def udf_intercept_hook(stage_name: Optional[str] = None, enable_run_before: bool = True, enable_run_after: bool = True):
+    """
+    Decorator that executes UDFs targeted at this stage.
+
+    This decorator integrates with the existing UDF system, providing full
+    UDF compilation, caching, and execution capabilities. UDFs can target
+    specific stages using run_before or run_after directives.
+
+    Args:
+        stage_name: Name of the stage (e.g., "image_dedup", "text_extract").
+                   If None, will attempt to use self.stage_name from the decorated method's instance.
+        enable_run_before: Whether to execute UDFs with run_before=True (default: True)
+        enable_run_after: Whether to execute UDFs with run_after=True (default: True)
+
+    Examples:
+        # Automatic stage name detection (recommended)
+        @traceable("image_deduplication")
+        @udf_intercept_hook()  # Uses self.stage_name automatically
+        @filter_by_task(required_tasks=["dedup"])
+        def on_data(self, control_message: IngestControlMessage) -> IngestControlMessage:
+            return control_message
+
+        # Explicit stage name (fallback/override)
+        @traceable("data_sink")
+        @udf_intercept_hook("data_sink", enable_run_after=False)
+        @filter_by_task(required_tasks=["store"])
+        def on_data(self, control_message: IngestControlMessage) -> IngestControlMessage:
+            return control_message
+
+        # Only run_after UDFs (e.g., for source stages)
+        @traceable("data_source")
+        @udf_intercept_hook(enable_run_before=False)  # Uses self.stage_name automatically
+        def on_data(self, control_message: IngestControlMessage) -> IngestControlMessage:
+            return control_message
+    """
+
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            # Check if UDF processing is globally disabled
+            if os.getenv("INGEST_DISABLE_UDF_PROCESSING"):
+                logger.debug("UDF processing is disabled via INGEST_DISABLE_UDF_PROCESSING environment variable")
+                return func(*args, **kwargs)
+
+            # Determine the stage name to use
+            resolved_stage_name = stage_name
+
+            # If no explicit stage_name provided, try to get it from self.stage_name
+            if resolved_stage_name is None and len(args) >= 1:
+                stage_instance = args[0]  # 'self' in method calls
+                if hasattr(stage_instance, "stage_name") and stage_instance.stage_name:
+                    resolved_stage_name = stage_instance.stage_name
+                    logger.debug(f"Using auto-detected stage name: '{resolved_stage_name}'")
+                else:
+                    logger.warning(
+                        "No stage_name provided and could not auto-detect from instance. Skipping UDF intercept."
+                    )
+                    return func(*args, **kwargs)
+            elif resolved_stage_name is None:
+                logger.warning(
+                    "No stage_name provided and no instance available for auto-detection. Skipping UDF intercept."
+                )
+                return func(*args, **kwargs)
+
+            # Extract control_message from args (handle both self.method and function cases)
+            control_message = None
+            if len(args) >= 2 and hasattr(args[1], "get_tasks"):
+                control_message = args[1]  # self.method case
+                args_list = list(args)
+            elif len(args) >= 1 and hasattr(args[0], "get_tasks"):
+                control_message = args[0]  # function case
+                args_list = list(args)
+
+            if control_message:
+                # Execute UDFs that should run before this stage (if enabled)
+                if enable_run_before:
+                    control_message = execute_targeted_udfs(control_message, resolved_stage_name, "run_before")
+                    # Update args with modified control_message
+                    if len(args) >= 2 and hasattr(args[1], "get_tasks"):
+                        args_list[1] = control_message
+                    else:
+                        args_list[0] = control_message
+
+                # Execute the original stage logic
+                result = func(*tuple(args_list), **kwargs)
+
+                # Execute UDFs that should run after this stage (if enabled)
+                if enable_run_after and hasattr(result, "get_tasks"):  # Result is control_message
+                    result = execute_targeted_udfs(result, resolved_stage_name, "run_after")
+
+                return result
+            else:
+                return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+def get_udf_cache_stats() -> Dict[str, Any]:
+    """Get UDF cache performance statistics"""
+    return _udf_cache.get_stats()

nv_ingest/framework/util/service/impl/ingest/redis_ingest_service.py

@@ -7,9 +7,7 @@ import json
 import logging
 import os
 from json import JSONDecodeError
-from typing import Optional, Dict, Any
-
-from typing import List
+from typing import Optional, Dict, Any, List
 
 import redis
 
@@ -133,6 +131,8 @@ class RedisIngestService(IngestServiceMeta):
         self._bulk_vdb_cache_prefix: str = "vdb_bulk_upload_cache:"
         self._cache_prefix: str = "processing_cache:"
         self._state_prefix: str = "job_state:"
+        # Bound async-to-thread concurrency slightly below Redis connection pool
+        self._async_operation_semaphore: Optional[asyncio.Semaphore] = None
 
         self._ingest_client = RedisClient(
             host=self._redis_hostname,
@@ -151,6 +151,16 @@ class RedisIngestService(IngestServiceMeta):
             f"FetchMode: {fetch_mode.name}, ResultTTL: {result_data_ttl_seconds}, StateTTL: {state_ttl_seconds}"
         )
 
+    def _get_async_semaphore(self) -> asyncio.Semaphore:
+        if self._async_operation_semaphore is None:
+            semaphore_limit = max(1, self._concurrency_level - 2)
+            self._async_operation_semaphore = asyncio.Semaphore(semaphore_limit)
+        return self._async_operation_semaphore
+
+    async def _run_bounded_to_thread(self, func, *args, **kwargs):
+        async with self._get_async_semaphore():
+            return await asyncio.to_thread(func, *args, **kwargs)
+
     async def submit_job(self, job_spec_wrapper: "MessageWrapper", trace_id: str) -> str:
         """
         Validates, prepares, and submits a job specification to the Redis task queue.
@@ -208,12 +218,33 @@ class RedisIngestService(IngestServiceMeta):
             ttl_for_result: Optional[int] = (
                 self._result_data_ttl_seconds if self._fetch_mode == FetchMode.NON_DESTRUCTIVE else None
             )
+            # Determine target queue based on optional QoS hint
+            queue_hint = None
+            try:
+                routing_opts = job_spec.get("routing_options") or {}
+                tracing_opts = job_spec.get("tracing_options") or {}
+                queue_hint = routing_opts.get("queue_hint") or tracing_opts.get("queue_hint")
+            except Exception:
+                queue_hint = None
+            allowed = {"default", "immediate", "micro", "small", "medium", "large"}
+            if isinstance(queue_hint, str) and queue_hint in allowed:
+                if queue_hint == "default":
+                    channel_name = self._redis_task_queue
+                else:
+                    channel_name = f"{self._redis_task_queue}_{queue_hint}"
+            else:
+                channel_name = self._redis_task_queue
+            logger.debug(
+                f"Submitting job {trace_id} to queue '{channel_name}' (hint={queue_hint}) "
+                f"with result TTL: {ttl_for_result}"
+            )
+
             logger.debug(
                 f"Submitting job {trace_id} to queue '{self._redis_task_queue}' with result TTL: {ttl_for_result}"
             )
-            await asyncio.to_thread(
+            await self._run_bounded_to_thread(
                 self._ingest_client.submit_message,
-                channel_name=self._redis_task_queue,
+                channel_name=channel_name,
                 message=job_spec_json,
                 ttl_seconds=ttl_for_result,
             )
@@ -252,7 +283,7 @@ class RedisIngestService(IngestServiceMeta):
         try:
             result_channel: str = f"{job_id}"
             logger.debug(f"Attempting to fetch job result for {job_id} using mode {self._fetch_mode.name}")
-            message = await asyncio.to_thread(
+            message = await self._run_bounded_to_thread(
                 self._ingest_client.fetch_message,
                 channel_name=result_channel,
                 timeout=10,
@@ -264,7 +295,7 @@ class RedisIngestService(IngestServiceMeta):
                 logger.warning(f"fetch_message for {job_id} returned None unexpectedly.")
                 raise TimeoutError("No data found (unexpected None response).")
         except (TimeoutError, redis.RedisError, ConnectionError, ValueError, RuntimeError) as e:
-            logger.info(f"Fetch operation for job {job_id} did not complete: ({type(e).__name__}) {e}")
+            logger.debug(f"Fetch operation for job {job_id} did not complete: ({type(e).__name__}) {e}")
             raise e
         except Exception as e:
             logger.exception(f"Unexpected error during async fetch_job for {job_id}: {e}")
@@ -289,7 +320,7 @@ class RedisIngestService(IngestServiceMeta):
         ttl_to_set: Optional[int] = self._state_ttl_seconds
         try:
             logger.debug(f"Setting state for {job_id} to {state} with TTL {ttl_to_set}")
-            await asyncio.to_thread(
+            await self._run_bounded_to_thread(
                 self._ingest_client.get_client().set,
                 state_key,
                 state,
@@ -317,7 +348,10 @@ class RedisIngestService(IngestServiceMeta):
         """
         state_key: str = f"{self._state_prefix}{job_id}"
         try:
-            data_bytes: Optional[bytes] = await asyncio.to_thread(self._ingest_client.get_client().get, state_key)
+            data_bytes: Optional[bytes] = await self._run_bounded_to_thread(
+                self._ingest_client.get_client().get,
+                state_key,
+            )
             if data_bytes:
                 state: str = data_bytes.decode("utf-8")
                 logger.debug(f"Retrieved state for {job_id}: {state}")
@@ -350,7 +384,7 @@ class RedisIngestService(IngestServiceMeta):
         cache_key: str = f"{self._cache_prefix}{job_id}"
         try:
             data_to_store: str = json.dumps([job.model_dump(mode="json") for job in jobs_data])
-            await asyncio.to_thread(
+            await self._run_bounded_to_thread(
                 self._ingest_client.get_client().set,
                 cache_key,
                 data_to_store,
@@ -375,7 +409,10 @@ class RedisIngestService(IngestServiceMeta):
         """
         cache_key: str = f"{self._cache_prefix}{job_id}"
         try:
-            data_bytes: Optional[bytes] = await asyncio.to_thread(self._ingest_client.get_client().get, cache_key)
+            data_bytes: Optional[bytes] = await self._run_bounded_to_thread(
+                self._ingest_client.get_client().get,
+                cache_key,
+            )
             if data_bytes is None:
                 return []
             return [ProcessingJob(**job) for job in json.loads(data_bytes)]
@@ -393,3 +430,170 @@ class RedisIngestService(IngestServiceMeta):
             The current fetch mode.
         """
         return self._fetch_mode
+
+    async def set_parent_job_mapping(
+        self,
+        parent_job_id: str,
+        subjob_ids: List[str],
+        metadata: Dict[str, Any],
+        *,
+        subjob_descriptors: Optional[List[Dict[str, Any]]] = None,
+    ) -> None:
+        """
+        Store parent-subjob mapping in Redis for V2 PDF splitting.
+
+        Parameters
+        ----------
+        parent_job_id : str
+            The parent job identifier
+        subjob_ids : List[str]
+            List of subjob identifiers
+        metadata : Dict[str, Any]
+            Metadata about the parent job (total_pages, original_source_id, etc.)
+        subjob_descriptors : List[Dict[str, Any]], optional
+            Detailed descriptors (job_id, chunk_index, start/end pages) for subjobs
+        """
+        parent_key = f"parent:{parent_job_id}:subjobs"
+        metadata_key = f"parent:{parent_job_id}:metadata"
+
+        try:
+            # Store subjob IDs as a set (only if there are subjobs)
+            if subjob_ids:
+                await self._run_bounded_to_thread(
+                    self._ingest_client.get_client().sadd,
+                    parent_key,
+                    *subjob_ids,
+                )
+
+            # Store metadata as hash (including original subjob ordering for deterministic fetches)
+            metadata_to_store = dict(metadata)
+            try:
+                metadata_to_store["subjob_order"] = json.dumps(subjob_ids)
+            except (TypeError, ValueError):
+                logger.warning(
+                    "Unable to serialize subjob ordering for parent %s; falling back to Redis set ordering",
+                    parent_job_id,
+                )
+                metadata_to_store.pop("subjob_order", None)
+
+            if subjob_descriptors:
+                metadata_to_store["subjob_descriptors"] = json.dumps(subjob_descriptors)
+
+            await self._run_bounded_to_thread(
+                self._ingest_client.get_client().hset,
+                metadata_key,
+                mapping=metadata_to_store,
+            )
+
+            # Set TTL on both keys to match state TTL
+            if self._state_ttl_seconds:
+                await self._run_bounded_to_thread(
+                    self._ingest_client.get_client().expire,
+                    parent_key,
+                    self._state_ttl_seconds,
+                )
+                await self._run_bounded_to_thread(
+                    self._ingest_client.get_client().expire,
+                    metadata_key,
+                    self._state_ttl_seconds,
+                )
+
+            logger.debug(f"Stored parent job mapping for {parent_job_id} with {len(subjob_ids)} subjobs")
+
+        except Exception as err:
+            logger.exception(f"Error storing parent job mapping for {parent_job_id}: {err}")
+            raise
+
+    async def get_parent_job_info(self, parent_job_id: str) -> Optional[Dict[str, Any]]:
+        """
+        Retrieve parent job information including subjob IDs and metadata.
+
+        Parameters
+        ----------
+        parent_job_id : str
+            The parent job identifier
+
+        Returns
+        -------
+        Dict[str, Any] or None
+            Dictionary with 'subjob_ids' and 'metadata' keys, or None if not a parent job
+        """
+        parent_key = f"parent:{parent_job_id}:subjobs"
+        metadata_key = f"parent:{parent_job_id}:metadata"
+
+        try:
+            # Check if this is a parent job (check metadata_key since non-split PDFs may not have parent_key)
+            exists = await self._run_bounded_to_thread(
+                self._ingest_client.get_client().exists,
+                metadata_key,  # Check metadata instead of parent_key for non-split PDF support
+            )
+
+            if not exists:
+                return None
+
+            # Get subjob IDs (may be empty for non-split PDFs)
+            subjob_ids_bytes = await self._run_bounded_to_thread(
+                self._ingest_client.get_client().smembers,
+                parent_key,
+            )
+            subjob_id_set = {id.decode("utf-8") for id in subjob_ids_bytes} if subjob_ids_bytes else set()
+
+            # Get metadata
+            metadata_dict = await self._run_bounded_to_thread(
+                self._ingest_client.get_client().hgetall,
+                metadata_key,
+            )
+            metadata = {k.decode("utf-8"): v.decode("utf-8") for k, v in metadata_dict.items()}
+
+            # Convert numeric strings back to numbers
+            if "total_pages" in metadata:
+                metadata["total_pages"] = int(metadata["total_pages"])
+            if "pages_per_chunk" in metadata:
+                try:
+                    metadata["pages_per_chunk"] = int(metadata["pages_per_chunk"])
+                except ValueError:
+                    metadata.pop("pages_per_chunk", None)
+
+            ordered_ids: Optional[List[str]] = None
+            stored_order = metadata.pop("subjob_order", None)
+            if stored_order:
+                try:
+                    candidate_order = json.loads(stored_order)
+                    if isinstance(candidate_order, list):
+                        ordered_ids = [sid for sid in candidate_order if sid in subjob_id_set]
+                except (ValueError, TypeError) as exc:
+                    logger.warning(
+                        "Failed to parse stored subjob order for parent %s: %s",
+                        parent_job_id,
+                        exc,
+                    )
+
+            if ordered_ids is None:
+                ordered_ids = sorted(subjob_id_set)
+            else:
+                remaining_ids = sorted(subjob_id_set - set(ordered_ids))
+                ordered_ids.extend(remaining_ids)
+
+            subjob_descriptors: Optional[List[Dict[str, Any]]] = None
+            stored_descriptors = metadata.pop("subjob_descriptors", None)
+            if stored_descriptors:
+                try:
+                    decoded = json.loads(stored_descriptors)
+                    if isinstance(decoded, list):
+                        subjob_descriptors = decoded
+                except (ValueError, TypeError) as exc:
+                    logger.warning(
+                        "Failed to parse stored subjob descriptors for parent %s: %s",
+                        parent_job_id,
+                        exc,
+                    )
+
+            return {
+                "subjob_ids": ordered_ids,
+                "metadata": metadata,
+                "subjob_descriptors": subjob_descriptors or [],
+            }
+
+        except Exception as err:
+            logger.error(f"Error retrieving parent job info for {parent_job_id}: {err}")
+            return None

nv_ingest/pipeline/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0

nv_ingest/pipeline/config/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0