nv-ingest 2025.10.4.dev20251004__py3-none-any.whl → 2025.12.10.dev20251210__py3-none-any.whl

nv_ingest/framework/orchestration/process/dependent_services.py

@@ -18,6 +18,18 @@ from nv_ingest_api.util.message_brokers.simple_message_broker.broker import Simp
 logger = logging.getLogger(__name__)
 
 
+def _broker_server_target(host, port, max_queue_size):
+    """
+    Target function to be run in a separate process for the SimpleMessageBroker.
+    """
+    server = SimpleMessageBroker(host, port, max_queue_size)
+    try:
+        server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    except Exception:
+        pass
+    server.serve_forever()
+
+
 def start_simple_message_broker(broker_client: dict) -> multiprocessing.Process:
     """
     Starts a SimpleMessageBroker server in a separate process.
@@ -58,16 +70,11 @@ def start_simple_message_broker(broker_client: dict) -> multiprocessing.Process:
             f"continuing to spawn a broker process (tests expect a Process to be returned)"
         )
 
-    def broker_server():
-        # Optionally, set socket options here for reuse (note: binding occurs in server __init__).
-        server = SimpleMessageBroker(server_host, server_port, max_queue_size)
-        try:
-            server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        except Exception:
-            pass
-        server.serve_forever()
-
-    p = multiprocessing.Process(target=broker_server)
+    p = multiprocessing.Process(
+        target=_broker_server_target,
+        args=(server_host, server_port, max_queue_size),
+        daemon=True,
+    )
     # If we're launching from inside the pipeline subprocess, mark daemon so the
     # broker dies automatically when the subprocess exits.
     p.daemon = os.environ.get("NV_INGEST_BROKER_IN_SUBPROCESS") == "1"

nv_ingest/framework/orchestration/process/strategies.py

@@ -11,9 +11,10 @@ Strategy pattern for clean separation of execution concerns.
 """
 
 import atexit
-import os
 import logging
 import multiprocessing
+import os
+import sys
 import time
 from abc import ABC, abstractmethod
 
@@ -132,7 +133,10 @@ class SubprocessStrategy(ProcessExecutionStrategy):
         logger.info("Launching pipeline in Python subprocess using multiprocessing.")
 
         # Create subprocess using fork context
-        ctx = multiprocessing.get_context("fork")
+        start_method = "fork"
+        if sys.platform.lower() == "darwin":
+            start_method = "spawn"
+        ctx = multiprocessing.get_context(start_method)
         process = ctx.Process(
             target=run_pipeline_process,
             args=(

nv_ingest/framework/orchestration/process/termination.py

@@ -19,20 +19,45 @@ logger = logging.getLogger(__name__)
 
 
 def _safe_log(level: int, msg: str) -> None:
-    """Best-effort logging that won't crash during interpreter shutdown."""
+    """Best-effort logging that won't emit handler tracebacks on closed streams.
+
+    Temporarily disables logging.raiseExceptions to prevent the logging module
+    from printing "--- Logging error ---" to stderr if a handler's stream is
+    already closed (common during process teardown). Falls back to writing to
+    sys.__stderr__ if available.
+    """
     try:
-        logger.log(level, msg)
+        import logging as _logging
+
+        prev = getattr(_logging, "raiseExceptions", True)
+        # Suppress handler errors being printed to stderr
+        _logging.raiseExceptions = False
+
+        # If there are no handlers, skip and use stderr fallback
+        if logger.handlers:
+            logger.log(level, msg)
+            return
     except Exception:
+        # Intentionally ignore and try stderr fallback
+        pass
+    finally:
         try:
-            # Fallback to stderr if available
-            import sys
+            import logging as _logging  # re-import safe even if earlier failed
 
-            if hasattr(sys, "__stderr__") and sys.__stderr__:
-                sys.__stderr__.write(msg + "\n")
-                sys.__stderr__.flush()
+            _logging.raiseExceptions = prev  # type: ignore[name-defined]
         except Exception:
             pass
 
+    # Fallback to stderr if available
+    try:
+        import sys
+
+        if hasattr(sys, "__stderr__") and sys.__stderr__:
+            sys.__stderr__.write(msg + "\n")
+            sys.__stderr__.flush()
+    except Exception:
+        pass
+
 
 def kill_pipeline_process_group(process) -> None:
     """
@@ -74,7 +99,17 @@ def kill_pipeline_process_group(process) -> None:
 
     try:
         # Send graceful termination to the entire process group
-        os.killpg(os.getpgid(pid), signal.SIGTERM)
+        try:
+            pgid = os.getpgid(pid)
+        except Exception:
+            # Process already gone
+            _safe_log(logging.DEBUG, f"Process group for PID {pid} not found during SIGTERM phase")
+            return
+        try:
+            os.killpg(pgid, signal.SIGTERM)
+        except ProcessLookupError:
+            _safe_log(logging.DEBUG, f"Process group for PID {pid} no longer exists (SIGTERM)")
+            return
 
         # If we have a Process handle, give it a chance to exit cleanly
         if proc is not None and hasattr(proc, "join"):
@@ -95,7 +130,12 @@ def kill_pipeline_process_group(process) -> None:
         if still_alive:
             _safe_log(logging.WARNING, "Process group did not terminate gracefully, using SIGKILL")
             try:
-                os.killpg(os.getpgid(pid), signal.SIGKILL)
+                try:
+                    pgid2 = os.getpgid(pid)
+                except Exception:
+                    _safe_log(logging.DEBUG, f"Process group for PID {pid} vanished before SIGKILL")
+                    return
+                os.killpg(pgid2, signal.SIGKILL)
             finally:
                 if proc is not None and hasattr(proc, "join"):
                     try:

nv_ingest/framework/orchestration/ray/examples/pipeline_test_harness.py

@@ -152,11 +152,11 @@ if __name__ == "__main__":
     os.environ["OCR_MODEL_NAME"] = "paddle"
     os.environ["NEMORETRIEVER_PARSE_HTTP_ENDPOINT"] = "https://integrate.api.nvidia.com/v1/chat/completions"
     os.environ["VLM_CAPTION_ENDPOINT"] = "https://integrate.api.nvidia.com/v1/chat/completions"
-    os.environ["VLM_CAPTION_MODEL_NAME"] = "nvidia/llama-3.1-nemotron-nano-vl-8b-v1"
+    os.environ["VLM_CAPTION_MODEL_NAME"] = "nvidia/nemotron-nano-12b-v2-vl"
     logger.info("Environment variables set.")
 
     image_caption_endpoint_url = "https://integrate.api.nvidia.com/v1/chat/completions"
-    model_name = "nvidia/llama-3.1-nemotron-nano-vl-8b-v1"
+    model_name = "nvidia/nemotron-nano-12b-v2-vl"
     yolox_grpc, yolox_http, yolox_auth, yolox_protocol = get_nim_service("yolox")
     (
         yolox_table_structure_grpc,

nv_ingest/framework/orchestration/ray/stages/extractors/audio_extractor.py

@@ -5,7 +5,6 @@
 
 import logging
 from typing import Optional
-
 import ray
 
 from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
@@ -67,7 +66,6 @@ class AudioExtractorStage(RayActorStage):
         # Extract the DataFrame payload.
         df_ledger = control_message.payload()
         self._logger.debug("Extracted payload with %d rows.", len(df_ledger))
-
         # Remove the "audio_data_extract" task from the message to obtain task-specific configuration.
         task_config = remove_task_by_type(control_message, "extract")
         self._logger.debug("Extracted task config: %s", sanitize_for_logging(task_config))

nv_ingest/framework/orchestration/ray/stages/extractors/ocr_extractor.py

@@ -0,0 +1,71 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import ray
+
+from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
+from nv_ingest.framework.util.flow_control import filter_by_task
+from nv_ingest_api.internal.extract.image.ocr_extractor import extract_text_data_from_image_internal
+from nv_ingest_api.internal.primitives.ingest_control_message import IngestControlMessage, remove_task_by_type
+from nv_ingest_api.internal.primitives.tracing.tagging import traceable, set_trace_timestamps_with_parent_context
+from nv_ingest_api.internal.schemas.extract.extract_ocr_schema import OCRExtractorSchema
+from nv_ingest_api.util.exception_handlers.decorators import nv_ingest_node_failure_try_except
+from typing import Optional
+
+from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
+
+logger = logging.getLogger(__name__)
+
+
+@ray.remote
+class OCRExtractorStage(RayActorStage):
+    """
+    A Ray actor stage that extracts text data from image content.
+
+    It expects an IngestControlMessage containing a DataFrame with image data. It then:
+      1. Removes the "text_data_extract" task from the message.
+      2. Calls the text extraction logic using a validated configuration.
+      3. Updates the message payload with the extracted text DataFrame.
+    """
+
+    def __init__(self, config: OCRExtractorSchema, stage_name: Optional[str] = None) -> None:
+        super().__init__(config, log_to_stdout=False, stage_name=stage_name)
+        try:
+            self.validated_config = config
+            self._logger.info("OCRExtractorStage configuration validated successfully.")
+        except Exception as e:
+            self._logger.exception(f"Error validating Text extractor config: {e}")
+            raise
+
+    @nv_ingest_node_failure_try_except()
+    @traceable()
+    @udf_intercept_hook()
+    @filter_by_task(required_tasks=["ocr_data_extract"])
+    def on_data(self, control_message: IngestControlMessage) -> IngestControlMessage:
+        # Extract DataFrame payload
+        df_ledger = control_message.payload()
+        if df_ledger.empty:
+            return control_message
+
+        # Remove the "text_data_extract" task from the message
+        task_config = remove_task_by_type(control_message, "ocr_data_extract")
+
+        execution_trace_log = {}
+        new_df, extraction_info = extract_text_data_from_image_internal(
+            df_extraction_ledger=df_ledger,
+            task_config=task_config,
+            extraction_config=self.validated_config,
+            execution_trace_log=execution_trace_log,
+        )
+
+        control_message.payload(new_df)
+        control_message.set_metadata("ocr_extraction_info", extraction_info)
+
+        do_trace_tagging = control_message.get_metadata("config::add_trace_tagging") is True
+        if do_trace_tagging and execution_trace_log:
+            parent_name = self.stage_name if self.stage_name else "ocr_extractor"
+            set_trace_timestamps_with_parent_context(control_message, execution_trace_log, parent_name, logger)
+
+        return control_message

nv_ingest/framework/orchestration/ray/stages/sources/message_broker_task_source.py

@@ -30,6 +30,7 @@ from nv_ingest_api.internal.schemas.meta.ingest_job_schema import validate_inges
 from nv_ingest_api.util.message_brokers.simple_message_broker.simple_client import SimpleClient
 from nv_ingest_api.util.service_clients.redis.redis_client import RedisClient
 from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
+from nv_ingest_api.util.message_brokers.qos_scheduler import QosScheduler
 
 logger = logging.getLogger(__name__)
 
@@ -89,8 +90,10 @@ class MessageBrokerTaskSourceConfig(BaseModel):
 
     # Use the discriminated union for broker_client
     broker_client: Union[RedisClientConfig, SimpleClientConfig] = Field(..., discriminator="client_type")
-    task_queue: str = Field(..., description="The name of the queue to fetch tasks from.")
-    poll_interval: float = Field(default=0.1, gt=0, description="Polling interval in seconds.")
+    task_queue: str = Field(
+        ..., description="The base name of the queue to fetch tasks from. Derives sub-queues for fair scheduling."
+    )
+    poll_interval: float = Field(default=0.0, gt=0, description="Polling interval in seconds.")
 
 
 @ray.remote
@@ -134,7 +137,29 @@ class MessageBrokerTaskSourceStage(RayActorSourceStage):
         self._current_backoff_sleep: float = 0.0
         self._last_backoff_log_time: float = 0.0
 
-        self._logger.debug("MessageBrokerTaskSourceStage initialized. Task queue: %s", self.task_queue)
+        # Initialize QoS scheduler. Use a simple base-queue strategy for SimpleClient.
+        strategy = "simple" if isinstance(self.client, SimpleClient) else "lottery"
+        self.scheduler = QosScheduler(
+            self.task_queue,
+            num_prefetch_threads=6,  # one per category (no-op for simple strategy)
+            total_buffer_capacity=96,  # e.g., ~16 per thread
+            prefetch_poll_interval=0.002,  # faster polling for responsiveness
+            prefetch_non_immediate=True,  # enable prefetch for non-immediate categories
+            strategy=strategy,
+        )
+
+        self._logger.info(
+            "MessageBrokerTaskSourceStage initialized. Base task queue: %s | Derived queues: %s",
+            self.task_queue,
+            {
+                "immediate": f"{self.task_queue}_immediate",
+                "micro": f"{self.task_queue}_micro",
+                "small": f"{self.task_queue}_small",
+                "medium": f"{self.task_queue}_medium",
+                "large": f"{self.task_queue}_large",
+                "default": f"{self.task_queue}",
+            },
+        )
 
     # --- Private helper methods ---
     def _create_client(self):
@@ -265,14 +290,21 @@ class MessageBrokerTaskSourceStage(RayActorSourceStage):
 
         return control_message
 
-    def _fetch_message(self, timeout=100):
+    def _fetch_message(self, timeout=0):
         """
-        Fetch a message from the message broker.
+        Fetch a message from the message broker using fair scheduling across derived queues.
+        This is a non-blocking sweep across all queues for the current scheduling cycle. If no
+        message is found across any queue, return None so the caller can sleep briefly.
         """
         try:
-            job = self.client.fetch_message(self.task_queue, timeout)
+            # Use scheduler to fetch next. In simple strategy this will block up to poll_interval on base queue.
+            job = self.scheduler.fetch_next(self.client, timeout=self.config.poll_interval)
             if job is None:
-                self._logger.debug("No message received from '%s'", self.task_queue)
+                self._logger.debug(
+                    "No message received from derived queues for base "
+                    "'%s' (immediate, micro, small, medium, large, default)",
+                    self.task_queue,
+                )
                 # Do not treat normal empty polls as failures
                 self._fetch_failure_count = 0
                 self._current_backoff_sleep = 0.0
@@ -336,7 +368,8 @@ class MessageBrokerTaskSourceStage(RayActorSourceStage):
         Instead of reading from an input edge, fetch a message from the broker.
         """
         self._logger.debug("read_input: calling _fetch_message()")
-        job = self._fetch_message(timeout=100)
+        # Perform a non-blocking sweep across all queues for this cycle
+        job = self._fetch_message(timeout=0)
         if job is None:
             # Sleep for either the configured poll interval or the current backoff, whichever is larger
             sleep_time = max(self.config.poll_interval, getattr(self, "_current_backoff_sleep", 0.0))

nv_ingest/framework/orchestration/ray/stages/storage/image_storage.py

@@ -3,7 +3,9 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import logging
+import os
 from typing import Dict, Any, Optional
+from urllib.parse import urlparse
 
 import pandas as pd
 import ray
@@ -26,7 +28,8 @@ logger = logging.getLogger(__name__)
 @ray.remote
 class ImageStorageStage(RayActorStage):
     """
-    A Ray actor stage that stores images or structured content in MinIO and updates metadata with storage URLs.
+    A Ray actor stage that stores images or structured content using an fsspec-compatible backend and updates
+    metadata with storage URLs.
 
     This stage uses the validated configuration (ImageStorageModuleSchema) to process and store the DataFrame
     payload and updates the control message accordingly.
@@ -69,8 +72,16 @@ class ImageStorageStage(RayActorStage):
         task_config = remove_task_by_type(control_message, "store")
         # logger.debug("ImageStorageStage: Task configuration extracted: %s", pprint.pformat(task_config))
 
-        store_structured: bool = task_config.get("structured", True)
-        store_unstructured: bool = task_config.get("images", False)
+        stage_defaults = {
+            "structured": self.validated_config.structured,
+            "images": self.validated_config.images,
+            "storage_uri": self.validated_config.storage_uri,
+            "storage_options": self.validated_config.storage_options,
+            "public_base_url": self.validated_config.public_base_url,
+        }
+
+        store_structured: bool = task_config.get("structured", stage_defaults["structured"])
+        store_unstructured: bool = task_config.get("images", stage_defaults["images"])
 
         content_types: Dict[Any, Any] = {}
         if store_structured:
@@ -80,14 +91,34 @@ class ImageStorageStage(RayActorStage):
             content_types[ContentTypeEnum.IMAGE] = store_unstructured
 
         params: Dict[str, Any] = task_config.get("params", {})
-        params["content_types"] = content_types
 
-        logger.debug(f"Processing storage task with parameters: {params}")
+        storage_uri = task_config.get("storage_uri") or params.get("storage_uri") or stage_defaults["storage_uri"]
+        storage_options = {
+            **(stage_defaults["storage_options"] or {}),
+            **(task_config.get("storage_options") or {}),
+            **params.get("storage_options", {}),
+        }
+        if "public_base_url" in task_config:
+            public_base_url = task_config["public_base_url"]
+        else:
+            public_base_url = params.get("public_base_url", stage_defaults["public_base_url"])
+
+        storage_options = self._inject_storage_defaults(storage_uri, storage_options)
+
+        storage_params: Dict[str, Any] = {
+            "content_types": content_types,
+            "storage_uri": storage_uri,
+            "storage_options": storage_options,
+        }
+        if public_base_url:
+            storage_params["public_base_url"] = public_base_url
+
+        logger.debug("Processing storage task with parameters: %s", storage_params)
 
         # Store images or structured content.
         df_storage_ledger: pd.DataFrame = store_images_to_minio_internal(
             df_storage_ledger=df_payload,
-            task_config=params,
+            task_config=storage_params,
             storage_config={},
             execution_trace_log=None,
         )
@@ -98,3 +129,38 @@ class ImageStorageStage(RayActorStage):
         control_message.payload(df_storage_ledger)
 
         return control_message
+
+    @staticmethod
+    def _inject_storage_defaults(storage_uri: str, storage_options: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Populate storage options for common backends (e.g., MinIO/S3) using environment defaults.
+        """
+        parsed_scheme = urlparse(storage_uri).scheme.lower()
+        merged_options: Dict[str, Any] = {k: v for k, v in storage_options.items() if v is not None}
+
+        if parsed_scheme not in {"s3", "s3a", "s3n"}:
+            return merged_options
+
+        def _set_if_absent(key: str, env_var: str) -> None:
+            if key not in merged_options and env_var in os.environ:
+                merged_options[key] = os.environ[env_var]
+
+        _set_if_absent("key", "MINIO_ACCESS_KEY")
+        _set_if_absent("secret", "MINIO_SECRET_KEY")
+        if "token" not in merged_options and os.environ.get("MINIO_SESSION_TOKEN"):
+            merged_options["token"] = os.environ["MINIO_SESSION_TOKEN"]
+
+        client_kwargs = dict(merged_options.get("client_kwargs", {}))
+        endpoint = os.environ.get("MINIO_INTERNAL_ADDRESS")
+        if not endpoint:
+            endpoint = "http://minio:9000"
+        if endpoint and not endpoint.startswith(("http://", "https://")):
+            endpoint = f"http://{endpoint}"
+        client_kwargs.setdefault("endpoint_url", endpoint)
+        region = os.environ.get("MINIO_REGION")
+        if region:
+            client_kwargs.setdefault("region_name", region)
+        if client_kwargs:
+            merged_options["client_kwargs"] = client_kwargs
+
+        return merged_options