nv-ingest 2025.8.16.dev20250816__py3-none-any.whl → 2025.11.21.dev20251121__py3-none-any.whl

nv_ingest/framework/orchestration/process/termination.py

@@ -0,0 +1,147 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Process termination utilities, isolated to avoid circular imports.
+
+This module provides functions to terminate a process and its entire process
+group safely, without depending on pipeline construction or Ray types.
+"""
+
+import logging
+import os
+import signal
+import time
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+def _safe_log(level: int, msg: str) -> None:
+    """Best-effort logging that won't emit handler tracebacks on closed streams.
+
+    Temporarily disables logging.raiseExceptions to prevent the logging module
+    from printing "--- Logging error ---" to stderr if a handler's stream is
+    already closed (common during process teardown). Falls back to writing to
+    sys.__stderr__ if available.
+    """
+    try:
+        import logging as _logging
+
+        prev = getattr(_logging, "raiseExceptions", True)
+        # Suppress handler errors being printed to stderr
+        _logging.raiseExceptions = False
+
+        # If there are no handlers, skip and use stderr fallback
+        if logger.handlers:
+            logger.log(level, msg)
+            return
+    except Exception:
+        # Intentionally ignore and try stderr fallback
+        pass
+    finally:
+        try:
+            import logging as _logging  # re-import safe even if earlier failed
+
+            _logging.raiseExceptions = prev  # type: ignore[name-defined]
+        except Exception:
+            pass
+
+    # Fallback to stderr if available
+    try:
+        import sys
+
+        if hasattr(sys, "__stderr__") and sys.__stderr__:
+            sys.__stderr__.write(msg + "\n")
+            sys.__stderr__.flush()
+    except Exception:
+        pass
+
+
+def kill_pipeline_process_group(process) -> None:
+    """
+    Kill a process and its entire process group.
+
+    Accepts either a multiprocessing.Process-like object exposing a ``pid`` attribute
+    or a raw PID integer. Sends SIGTERM to the process group first, and escalates
+    to SIGKILL if it does not terminate within a short grace period.
+
+    Parameters
+    ----------
+    process : multiprocessing.Process | int
+        Process handle (or a raw PID int) for the process whose process group should be terminated.
+    """
+    proc: Optional[object] = None
+    pid: Optional[int] = None
+
+    if isinstance(process, int):
+        pid = process
+    elif hasattr(process, "pid"):
+        proc = process
+        try:
+            pid = int(getattr(proc, "pid"))
+        except Exception as e:
+            raise AttributeError(f"Invalid process-like object without usable pid: {e}")
+    else:
+        raise AttributeError(
+            "kill_pipeline_process_group expects a multiprocessing.Process or a PID int (process-like object with .pid)"
+        )
+
+    if proc is not None and hasattr(proc, "is_alive") and not proc.is_alive():
+        _safe_log(logging.DEBUG, "Process already terminated")
+        return
+
+    if pid is None:
+        raise AttributeError("Unable to determine PID for process group termination")
+
+    _safe_log(logging.INFO, f"Terminating pipeline process group (PID: {pid})")
+
+    try:
+        # Send graceful termination to the entire process group
+        try:
+            pgid = os.getpgid(pid)
+        except Exception:
+            # Process already gone
+            _safe_log(logging.DEBUG, f"Process group for PID {pid} not found during SIGTERM phase")
+            return
+        try:
+            os.killpg(pgid, signal.SIGTERM)
+        except ProcessLookupError:
+            _safe_log(logging.DEBUG, f"Process group for PID {pid} no longer exists (SIGTERM)")
+            return
+
+        # If we have a Process handle, give it a chance to exit cleanly
+        if proc is not None and hasattr(proc, "join"):
+            try:
+                proc.join(timeout=5.0)
+            except Exception:
+                pass
+            still_alive = getattr(proc, "is_alive", lambda: True)()
+        else:
+            # Without a handle, provide a small grace period
+            time.sleep(2.0)
+            try:
+                _ = os.getpgid(pid)
+                still_alive = True
+            except Exception:
+                still_alive = False
+
+        if still_alive:
+            _safe_log(logging.WARNING, "Process group did not terminate gracefully, using SIGKILL")
+            try:
+                try:
+                    pgid2 = os.getpgid(pid)
+                except Exception:
+                    _safe_log(logging.DEBUG, f"Process group for PID {pid} vanished before SIGKILL")
+                    return
+                os.killpg(pgid2, signal.SIGKILL)
+            finally:
+                if proc is not None and hasattr(proc, "join"):
+                    try:
+                        proc.join(timeout=3.0)
+                    except Exception:
+                        pass
+
+    except (ProcessLookupError, OSError) as e:
+        _safe_log(logging.DEBUG, f"Process group already terminated or not found: {e}")

nv_ingest/framework/orchestration/ray/examples/pipeline_test_harness.py

@@ -152,11 +152,11 @@ if __name__ == "__main__":
     os.environ["OCR_MODEL_NAME"] = "paddle"
     os.environ["NEMORETRIEVER_PARSE_HTTP_ENDPOINT"] = "https://integrate.api.nvidia.com/v1/chat/completions"
     os.environ["VLM_CAPTION_ENDPOINT"] = "https://integrate.api.nvidia.com/v1/chat/completions"
-    os.environ["VLM_CAPTION_MODEL_NAME"] = "nvidia/llama-3.1-nemotron-nano-vl-8b-v1"
+    os.environ["VLM_CAPTION_MODEL_NAME"] = "nvidia/nemotron-nano-12b-v2-vl"
     logger.info("Environment variables set.")
 
     image_caption_endpoint_url = "https://integrate.api.nvidia.com/v1/chat/completions"
-    model_name = "nvidia/llama-3.1-nemotron-nano-vl-8b-v1"
+    model_name = "nvidia/nemotron-nano-12b-v2-vl"
     yolox_grpc, yolox_http, yolox_auth, yolox_protocol = get_nim_service("yolox")
     (
         yolox_table_structure_grpc,

nv_ingest/framework/orchestration/ray/primitives/ray_pipeline.py

@@ -3,8 +3,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import multiprocessing
-import os
-import signal
 import threading
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
@@ -22,6 +20,7 @@ import logging
 import time
 
 from nv_ingest.framework.orchestration.ray.primitives.pipeline_topology import PipelineTopology, StageInfo
+from nv_ingest.framework.orchestration.process.termination import kill_pipeline_process_group
 from nv_ingest.framework.orchestration.ray.primitives.ray_stat_collector import RayStatsCollector
 from nv_ingest.framework.orchestration.ray.util.pipeline.pid_controller import PIDController, ResourceConstraintManager
 from nv_ingest.framework.orchestration.ray.util.pipeline.tools import wrap_callable_as_stage
@@ -120,24 +119,19 @@ class RayPipelineSubprocessInterface(PipelineInterface):
 
     def stop(self) -> None:
         """
-        Stops the subprocess pipeline. Tries terminate(), then escalates to SIGKILL on the process group if needed.
+        Stops the subprocess pipeline and its entire process group to ensure
+        any child processes (e.g., the simple message broker) are terminated.
         """
-        if not self._process.is_alive():
+        try:
+            pid = int(self._process.pid)
+        except Exception:
             return
 
+        # Always attempt to terminate the entire process group
         try:
-            self._process.terminate()
-            self._process.join(timeout=5.0)
+            kill_pipeline_process_group(pid)
         except Exception as e:
-            logger.warning(f"Failed to terminate process cleanly: {e}")
-
-        if self._process.is_alive():
-            try:
-                pgid = os.getpgid(self._process.pid)
-                os.killpg(pgid, signal.SIGKILL)
-            except Exception as e:
-                logger.error(f"Failed to force-kill process group: {e}")
-            self._process.join(timeout=3.0)
+            logger.warning(f"kill_pipeline_process_group failed: {e}")
 
 
 class RayPipelineInterface(PipelineInterface):

nv_ingest/framework/orchestration/ray/stages/extractors/audio_extractor.py

@@ -5,7 +5,6 @@
 
 import logging
 from typing import Optional
-
 import ray
 
 from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
@@ -17,6 +16,7 @@ from nv_ingest_api.internal.schemas.extract.extract_audio_schema import AudioExt
 from nv_ingest_api.util.exception_handlers.decorators import (
     nv_ingest_node_failure_try_except,
 )
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
 
@@ -66,10 +66,9 @@ class AudioExtractorStage(RayActorStage):
         # Extract the DataFrame payload.
         df_ledger = control_message.payload()
         self._logger.debug("Extracted payload with %d rows.", len(df_ledger))
-
         # Remove the "audio_data_extract" task from the message to obtain task-specific configuration.
         task_config = remove_task_by_type(control_message, "extract")
-        self._logger.debug("Extracted task config: %s", task_config)
+        self._logger.debug("Extracted task config: %s", sanitize_for_logging(task_config))
 
         # Perform audio text extraction.
         new_df, extraction_info = extract_text_from_audio_internal(

nv_ingest/framework/orchestration/ray/stages/extractors/chart_extractor.py

@@ -13,8 +13,11 @@ from nv_ingest.framework.util.flow_control import filter_by_task
 from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
 from nv_ingest_api.internal.primitives.tracing.tagging import traceable
 from nv_ingest_api.internal.schemas.extract.extract_chart_schema import ChartExtractorSchema
-from nv_ingest_api.util.exception_handlers.decorators import nv_ingest_node_failure_try_except
 from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
+from nv_ingest_api.util.exception_handlers.decorators import (
+    nv_ingest_node_failure_try_except,
+)
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 logger = logging.getLogger(__name__)
 
@@ -66,7 +69,7 @@ class ChartExtractorStage(RayActorStage):
 
         # Remove the "chart_data_extract" task to obtain task-specific configuration.
         task_config = remove_task_by_type(control_message, "chart_data_extract")
-        logger.debug("ChartExtractorStage: Task config extracted: %s", task_config)
+        logger.debug("ChartExtractorStage: Task config extracted: %s", sanitize_for_logging(task_config))
 
         # Perform chart data extraction.
         execution_trace_log = {}

nv_ingest/framework/orchestration/ray/stages/extractors/docx_extractor.py

@@ -16,6 +16,7 @@ from nv_ingest_api.internal.schemas.extract.extract_docx_schema import DocxExtra
 from nv_ingest_api.util.exception_handlers.decorators import (
     nv_ingest_node_failure_try_except,
 )
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
 
@@ -68,7 +69,7 @@ class DocxExtractorStage(RayActorStage):
 
         # Remove the "docx-extract" task from the message to obtain task-specific configuration.
         task_config = remove_task_by_type(control_message, "extract")
-        self._logger.debug("Extracted task config: %s", task_config)
+        self._logger.debug("Extracted task config: %s", sanitize_for_logging(task_config))
 
         # Perform DOCX content extraction.
         new_df, extraction_info = extract_primitives_from_docx_internal(

nv_ingest/framework/orchestration/ray/stages/extractors/html_extractor.py

@@ -17,6 +17,7 @@ from nv_ingest_api.internal.schemas.extract.extract_html_schema import HtmlExtra
 from nv_ingest_api.util.exception_handlers.decorators import (
     nv_ingest_node_failure_try_except,
 )
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
 
@@ -69,7 +70,7 @@ class HtmlExtractorStage(RayActorStage):
 
         # Remove the "html_content_extract" task from the message to obtain task-specific configuration.
         task_config = remove_task_by_type(control_message, "extract")
-        self._logger.debug("Extracted task config: %s", task_config)
+        self._logger.debug("Extracted task config: %s", sanitize_for_logging(task_config))
 
         # Perform html content extraction.
         new_df, extraction_info = extract_markdown_from_html_internal(

nv_ingest/framework/orchestration/ray/stages/extractors/image_extractor.py

@@ -16,6 +16,7 @@ from nv_ingest_api.internal.schemas.extract.extract_image_schema import ImageExt
 from nv_ingest_api.util.exception_handlers.decorators import (
     nv_ingest_node_failure_try_except,
 )
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
 
@@ -68,7 +69,7 @@ class ImageExtractorStage(RayActorStage):
 
             # Remove the "extract" task from the message to obtain task-specific configuration.
             task_config = remove_task_by_type(control_message, "extract")
-            logger.debug("Extracted task config: %s", task_config)
+            logger.debug("Extracted task config: %s", sanitize_for_logging(task_config))
 
             # Perform image primitives extraction.
             new_df, extraction_info = extract_primitives_from_image_internal(

nv_ingest/framework/orchestration/ray/stages/extractors/ocr_extractor.py

@@ -0,0 +1,71 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import ray
+
+from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
+from nv_ingest.framework.util.flow_control import filter_by_task
+from nv_ingest_api.internal.extract.image.ocr_extractor import extract_text_data_from_image_internal
+from nv_ingest_api.internal.primitives.ingest_control_message import IngestControlMessage, remove_task_by_type
+from nv_ingest_api.internal.primitives.tracing.tagging import traceable, set_trace_timestamps_with_parent_context
+from nv_ingest_api.internal.schemas.extract.extract_ocr_schema import OCRExtractorSchema
+from nv_ingest_api.util.exception_handlers.decorators import nv_ingest_node_failure_try_except
+from typing import Optional
+
+from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
+
+logger = logging.getLogger(__name__)
+
+
+@ray.remote
+class OCRExtractorStage(RayActorStage):
+    """
+    A Ray actor stage that extracts text data from image content.
+
+    It expects an IngestControlMessage containing a DataFrame with image data. It then:
+      1. Removes the "text_data_extract" task from the message.
+      2. Calls the text extraction logic using a validated configuration.
+      3. Updates the message payload with the extracted text DataFrame.
+    """
+
+    def __init__(self, config: OCRExtractorSchema, stage_name: Optional[str] = None) -> None:
+        super().__init__(config, log_to_stdout=False, stage_name=stage_name)
+        try:
+            self.validated_config = config
+            self._logger.info("OCRExtractorStage configuration validated successfully.")
+        except Exception as e:
+            self._logger.exception(f"Error validating Text extractor config: {e}")
+            raise
+
+    @nv_ingest_node_failure_try_except()
+    @traceable()
+    @udf_intercept_hook()
+    @filter_by_task(required_tasks=["ocr_data_extract"])
+    def on_data(self, control_message: IngestControlMessage) -> IngestControlMessage:
+        # Extract DataFrame payload
+        df_ledger = control_message.payload()
+        if df_ledger.empty:
+            return control_message
+
+        # Remove the "text_data_extract" task from the message
+        task_config = remove_task_by_type(control_message, "ocr_data_extract")
+
+        execution_trace_log = {}
+        new_df, extraction_info = extract_text_data_from_image_internal(
+            df_extraction_ledger=df_ledger,
+            task_config=task_config,
+            extraction_config=self.validated_config,
+            execution_trace_log=execution_trace_log,
+        )
+
+        control_message.payload(new_df)
+        control_message.set_metadata("ocr_extraction_info", extraction_info)
+
+        do_trace_tagging = control_message.get_metadata("config::add_trace_tagging") is True
+        if do_trace_tagging and execution_trace_log:
+            parent_name = self.stage_name if self.stage_name else "ocr_extractor"
+            set_trace_timestamps_with_parent_context(control_message, execution_trace_log, parent_name, logger)
+
+        return control_message

nv_ingest/framework/orchestration/ray/stages/extractors/pdf_extractor.py

@@ -15,7 +15,10 @@ from nv_ingest_api.internal.primitives.tracing.tagging import set_trace_timestam
 from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
 from nv_ingest.framework.util.flow_control import filter_by_task
 from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
-from nv_ingest_api.util.exception_handlers.decorators import nv_ingest_node_failure_try_except
+from nv_ingest_api.util.exception_handlers.decorators import (
+    nv_ingest_node_failure_try_except,
+)
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 logger = logging.getLogger(__name__)
 
@@ -87,7 +90,7 @@ class PDFExtractorStage(RayActorStage):
 
         # Remove the "extract" task from the message to obtain task-specific configuration.
         task_config = remove_task_by_type(control_message, "extract")
-        logger.debug("Extracted task config: %s", task_config)
+        logger.debug("Extracted task config: %s", sanitize_for_logging(task_config))
 
         # Perform PDF extraction.
         execution_trace_log = {}

nv_ingest/framework/orchestration/ray/stages/extractors/table_extractor.py

@@ -16,6 +16,7 @@ from nv_ingest_api.internal.schemas.extract.extract_table_schema import TableExt
 from nv_ingest_api.util.exception_handlers.decorators import (
     nv_ingest_node_failure_try_except,
 )
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 logger = logging.getLogger(__name__)
 
@@ -65,7 +66,7 @@ class TableExtractorStage(RayActorStage):
 
         # Remove the "table_data_extract" task to obtain task-specific configuration.
         task_config = remove_task_by_type(control_message, "table_data_extract")
-        logger.debug("Extracted task configuration: %s", task_config)
+        logger.debug("Extracted task configuration: %s", sanitize_for_logging(task_config))
 
         # Perform table data extraction.
         execution_trace_log = {}

nv_ingest/framework/orchestration/ray/stages/injectors/metadata_injector.py

@@ -25,6 +25,7 @@ from nv_ingest_api.util.exception_handlers.decorators import (
     nv_ingest_node_failure_try_except,
 )
 from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 logger = logging.getLogger(__name__)
 
@@ -42,7 +43,7 @@ class MetadataInjectionStage(RayActorStage):
         # Call the base initializer to set attributes like self._running.
         super().__init__(config, stage_name=stage_name)
         # Additional initialization can be added here if necessary.
-        self._logger.debug("MetadataInjectionStage initialized with config: %s", config)
+        self._logger.debug("MetadataInjectionStage initialized with config: %s", sanitize_for_logging(config))
 
     @nv_ingest_node_failure_try_except()
     @traceable()

nv_ingest/framework/orchestration/ray/stages/mutate/image_dedup.py

@@ -18,6 +18,7 @@ from nv_ingest_api.internal.schemas.mutate.mutate_image_dedup_schema import Imag
 from nv_ingest_api.util.exception_handlers.decorators import (
     nv_ingest_node_failure_try_except,
 )
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 logger = logging.getLogger(__name__)
 
@@ -68,7 +69,7 @@ class ImageDedupStage(RayActorStage):
 
             # Remove the "dedup" task from the message to obtain task-specific configuration.
             task_config = remove_task_by_type(control_message, "dedup")
-            logger.debug("Extracted task config: %s", task_config)
+            logger.debug("Extracted task config: %s", sanitize_for_logging(task_config))
 
             # Perform image deduplication.
             new_df = deduplicate_images_internal(

nv_ingest/framework/orchestration/ray/stages/mutate/image_filter.py

@@ -17,6 +17,7 @@ from nv_ingest_api.internal.schemas.transform.transform_image_filter_schema impo
 from nv_ingest_api.util.exception_handlers.decorators import (
     nv_ingest_node_failure_try_except,
 )
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 logger = logging.getLogger(__name__)
 
@@ -67,7 +68,7 @@ class ImageFilterStage(RayActorStage):
 
         # Remove the "filter" task from the message to obtain task-specific configuration.
         task_config = remove_task_by_type(control_message, "filter")
-        logger.debug("Extracted task config: %s", task_config)
+        logger.debug("Extracted task config: %s", sanitize_for_logging(task_config))
 
         task_params: Dict[str, Any] = task_config.get("params", {})
 

nv_ingest/framework/orchestration/ray/stages/sources/message_broker_task_source.py

@@ -29,6 +29,8 @@ from nv_ingest_api.internal.schemas.meta.ingest_job_schema import validate_inges
 # Import clients
 from nv_ingest_api.util.message_brokers.simple_message_broker.simple_client import SimpleClient
 from nv_ingest_api.util.service_clients.redis.redis_client import RedisClient
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
+from nv_ingest_api.util.message_brokers.qos_scheduler import QosScheduler
 
 logger = logging.getLogger(__name__)
 
@@ -88,8 +90,10 @@ class MessageBrokerTaskSourceConfig(BaseModel):
 
     # Use the discriminated union for broker_client
     broker_client: Union[RedisClientConfig, SimpleClientConfig] = Field(..., discriminator="client_type")
-    task_queue: str = Field(..., description="The name of the queue to fetch tasks from.")
-    poll_interval: float = Field(default=0.1, gt=0, description="Polling interval in seconds.")
+    task_queue: str = Field(
+        ..., description="The base name of the queue to fetch tasks from. Derives sub-queues for fair scheduling."
+    )
+    poll_interval: float = Field(default=0.0, gt=0, description="Polling interval in seconds.")
 
 
 @ray.remote
@@ -104,8 +108,11 @@ class MessageBrokerTaskSourceStage(RayActorSourceStage):
     def __init__(self, config: MessageBrokerTaskSourceConfig, stage_name: Optional[str] = None) -> None:
         super().__init__(config, log_to_stdout=False, stage_name=stage_name)
         self.config: MessageBrokerTaskSourceConfig  # Add a type hint for self.config
+
+        # Sanitize config before logging to avoid leaking secrets
+        _sanitized = sanitize_for_logging(config)
         self._logger.debug(
-            "Initializing MessageBrokerTaskSourceStage with config: %s", config.model_dump()
+            "Initializing MessageBrokerTaskSourceStage with config: %s", _sanitized
         )  # Log validated config
 
         # Access validated configuration directly via self.config
@@ -130,7 +137,29 @@ class MessageBrokerTaskSourceStage(RayActorSourceStage):
         self._current_backoff_sleep: float = 0.0
         self._last_backoff_log_time: float = 0.0
 
-        self._logger.debug("MessageBrokerTaskSourceStage initialized. Task queue: %s", self.task_queue)
+        # Initialize QoS scheduler. Use a simple base-queue strategy for SimpleClient.
+        strategy = "simple" if isinstance(self.client, SimpleClient) else "lottery"
+        self.scheduler = QosScheduler(
+            self.task_queue,
+            num_prefetch_threads=6,  # one per category (no-op for simple strategy)
+            total_buffer_capacity=96,  # e.g., ~16 per thread
+            prefetch_poll_interval=0.002,  # faster polling for responsiveness
+            prefetch_non_immediate=True,  # enable prefetch for non-immediate categories
+            strategy=strategy,
+        )
+
+        self._logger.info(
+            "MessageBrokerTaskSourceStage initialized. Base task queue: %s | Derived queues: %s",
+            self.task_queue,
+            {
+                "immediate": f"{self.task_queue}_immediate",
+                "micro": f"{self.task_queue}_micro",
+                "small": f"{self.task_queue}_small",
+                "medium": f"{self.task_queue}_medium",
+                "large": f"{self.task_queue}_large",
+                "default": f"{self.task_queue}",
+            },
+        )
 
     # --- Private helper methods ---
     def _create_client(self):
@@ -261,14 +290,21 @@ class MessageBrokerTaskSourceStage(RayActorSourceStage):
 
         return control_message
 
-    def _fetch_message(self, timeout=100):
+    def _fetch_message(self, timeout=0):
         """
-        Fetch a message from the message broker.
+        Fetch a message from the message broker using fair scheduling across derived queues.
+        This is a non-blocking sweep across all queues for the current scheduling cycle. If no
+        message is found across any queue, return None so the caller can sleep briefly.
         """
         try:
-            job = self.client.fetch_message(self.task_queue, timeout)
+            # Use scheduler to fetch next. In simple strategy this will block up to poll_interval on base queue.
+            job = self.scheduler.fetch_next(self.client, timeout=self.config.poll_interval)
             if job is None:
-                self._logger.debug("No message received from '%s'", self.task_queue)
+                self._logger.debug(
+                    "No message received from derived queues for base "
+                    "'%s' (immediate, micro, small, medium, large, default)",
+                    self.task_queue,
+                )
                 # Do not treat normal empty polls as failures
                 self._fetch_failure_count = 0
                 self._current_backoff_sleep = 0.0
@@ -332,7 +368,8 @@ class MessageBrokerTaskSourceStage(RayActorSourceStage):
         Instead of reading from an input edge, fetch a message from the broker.
         """
         self._logger.debug("read_input: calling _fetch_message()")
-        job = self._fetch_message(timeout=100)
+        # Perform a non-blocking sweep across all queues for this cycle
+        job = self._fetch_message(timeout=0)
         if job is None:
             # Sleep for either the configured poll interval or the current backoff, whichever is larger
             sleep_time = max(self.config.poll_interval, getattr(self, "_current_backoff_sleep", 0.0))

nv_ingest/framework/orchestration/ray/stages/storage/store_embeddings.py

@@ -16,6 +16,7 @@ from nv_ingest_api.internal.store.embed_text_upload import store_text_embeddings
 from nv_ingest_api.util.exception_handlers.decorators import (
     nv_ingest_node_failure_try_except,
 )
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
 
@@ -68,7 +69,7 @@ class EmbeddingStorageStage(RayActorStage):
 
         # Remove the "store_embedding" task from the message to obtain task-specific configuration.
         task_config = remove_task_by_type(control_message, "store_embedding")
-        logger.debug("Extracted task config: %s", task_config)
+        logger.debug("Extracted task config: %s", sanitize_for_logging(task_config))
 
         # Perform embedding storage.
         new_df = store_text_embeddings_internal(

nv_ingest/framework/orchestration/ray/stages/transforms/image_caption.py

@@ -18,6 +18,7 @@ from nv_ingest_api.internal.transform.caption_image import transform_image_creat
 from nv_ingest_api.util.exception_handlers.decorators import (
     nv_ingest_node_failure_try_except,
 )
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 logger = logging.getLogger(__name__)
 
@@ -67,7 +68,10 @@ class ImageCaptionTransformStage(RayActorStage):
 
         # Remove the "caption" task to obtain task-specific configuration.
         task_config = remove_task_by_type(control_message, "caption")
-        logger.debug("ImageCaptionTransformStage: Task configuration extracted: %s", pprint.pformat(task_config))
+        logger.debug(
+            "ImageCaptionTransformStage: Task configuration extracted: %s",
+            pprint.pformat(sanitize_for_logging(task_config)),
+        )
 
         # Call the caption extraction function.
         new_df = transform_image_create_vlm_caption_internal(

nv_ingest/framework/orchestration/ray/stages/transforms/text_embed.py

@@ -15,6 +15,7 @@ from nv_ingest_api.internal.transform.embed_text import transform_create_text_em
 from nv_ingest_api.util.exception_handlers.decorators import (
     nv_ingest_node_failure_try_except,
 )
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
 
@@ -62,7 +63,10 @@ class TextEmbeddingTransformStage(RayActorStage):
 
         # Remove the "embed" task to obtain task-specific configuration.
         task_config = remove_task_by_type(control_message, "embed")
-        self._logger.debug("TextEmbeddingTransformStage: Task configuration extracted: %s", pprint.pformat(task_config))
+        self._logger.debug(
+            "TextEmbeddingTransformStage: Task configuration extracted: %s",
+            pprint.pformat(sanitize_for_logging(task_config)),
+        )
 
         # Call the text embedding extraction function.
         new_df, execution_trace_log = transform_create_text_embeddings_internal(