nv-ingest 2025.8.4.dev20250804__py3-none-any.whl → 2025.12.10.dev20251210__py3-none-any.whl

nv_ingest/framework/orchestration/process/lifecycle.py

@@ -0,0 +1,214 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Pipeline lifecycle management for declarative execution.
+
+This module provides high-level lifecycle management for pipelines,
+orchestrating configuration resolution, broker setup, and execution
+using the configured strategy pattern.
+"""
+
+import logging
+import atexit
+import multiprocessing
+import os
+import signal
+from typing import Optional
+
+from nv_ingest.pipeline.pipeline_schema import PipelineConfigSchema
+from nv_ingest.framework.orchestration.execution.options import ExecutionOptions, ExecutionResult
+from nv_ingest.framework.orchestration.process.strategies import ProcessExecutionStrategy
+from nv_ingest.framework.orchestration.process.strategies import SubprocessStrategy
+from nv_ingest.framework.orchestration.process.dependent_services import start_simple_message_broker
+
+logger = logging.getLogger(__name__)
+
+
+class PipelineLifecycleManager:
+    """
+    High-level manager for pipeline lifecycle operations.
+
+    This class orchestrates the complete pipeline lifecycle including
+    broker setup, configuration validation, and execution using the
+    configured execution strategy.
+
+    Attributes
+    ----------
+    strategy : ProcessExecutionStrategy
+        The execution strategy to use for running pipelines.
+    """
+
+    def __init__(self, strategy: ProcessExecutionStrategy):
+        """
+        Initialize the lifecycle manager with an execution strategy.
+
+        Parameters
+        ----------
+        strategy : ProcessExecutionStrategy
+            The strategy to use for pipeline execution.
+        """
+        self.strategy = strategy
+        # Track broker process so we can terminate it during teardown
+        self._broker_process: Optional[multiprocessing.Process] = None
+
+    def start(self, config: PipelineConfigSchema, options: ExecutionOptions) -> ExecutionResult:
+        """
+        Start a pipeline using the configured execution strategy.
+
+        This method handles the complete pipeline startup process:
+        1. Validate configuration
+        2. Start message broker if required
+        3. Execute pipeline using the configured strategy
+
+        Parameters
+        ----------
+        config : PipelineConfigSchema
+            Validated pipeline configuration to execute.
+        options : ExecutionOptions
+            Execution options controlling blocking behavior and output.
+
+        Returns
+        -------
+        ExecutionResult
+            Result containing pipeline interface and/or timing information.
+
+        Raises
+        ------
+        RuntimeError
+            If pipeline startup fails.
+        """
+        logger.info("Starting pipeline lifecycle")
+
+        # If running pipeline in a subprocess and broker is enabled, ensure the broker
+        # is launched in the child process group by signaling via environment variable
+        prev_env = None
+        set_env = False
+        if getattr(config, "pipeline", None) and getattr(config.pipeline, "launch_simple_broker", False):
+            if isinstance(self.strategy, SubprocessStrategy):
+                prev_env = os.environ.get("NV_INGEST_BROKER_IN_SUBPROCESS")
+                os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"] = "1"
+                set_env = True
+
+        try:
+            # Start message broker if configured (may defer to subprocess based on env)
+            self._setup_message_broker(config)
+
+            # Execute pipeline using the configured strategy
+            result = self.strategy.execute(config, options)
+
+            logger.info("Pipeline lifecycle started successfully")
+            return result
+
+        except Exception as e:
+            logger.error(f"Failed to start pipeline lifecycle: {e}")
+            raise RuntimeError(f"Pipeline startup failed: {e}") from e
+        finally:
+            if set_env:
+                if prev_env is None:
+                    try:
+                        del os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"]
+                    except KeyError:
+                        pass
+                else:
+                    os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"] = prev_env
+
+    def _setup_message_broker(self, config: PipelineConfigSchema) -> None:
+        """
+        Set up message broker if required by configuration.
+
+        Parameters
+        ----------
+        config : PipelineConfigSchema
+            Pipeline configuration containing broker settings.
+        """
+        if config.pipeline.launch_simple_broker:
+            # If requested to launch broker inside the subprocess, skip here
+            if os.environ.get("NV_INGEST_BROKER_IN_SUBPROCESS") == "1":
+                logger.info("Deferring SimpleMessageBroker launch to subprocess")
+                return
+            logger.info("Starting simple message broker")
+            # Start the broker and retain a handle for cleanup.
+            # Use defaults (host=0.0.0.0, port=7671) as set by the broker implementation.
+            try:
+                self._broker_process = start_simple_message_broker({})
+                # Ensure cleanup at interpreter shutdown in case caller forgets
+                atexit.register(self._terminate_broker_atexit)
+                logger.info(f"SimpleMessageBroker started (pid={getattr(self._broker_process, 'pid', None)})")
+            except Exception as e:
+                logger.error(f"Failed to start SimpleMessageBroker: {e}")
+                raise
+        else:
+            logger.debug("Simple broker launch not required")
+
+    def stop(self, pipeline_id: Optional[str] = None) -> None:
+        """
+        Stop a running pipeline.
+
+        This method provides a hook for future pipeline stopping functionality.
+        Currently, pipeline stopping is handled by the individual interfaces.
+        Additionally, it ensures any dependent services (like the simple
+        message broker) are terminated to avoid lingering processes.
+
+        Parameters
+        ----------
+        pipeline_id : Optional[str]
+            Identifier of the pipeline to stop. Currently unused.
+        """
+        logger.info("Pipeline stop requested")
+        # Best-effort termination of broker if we started one
+        self._terminate_broker()
+
+    # --- Internal helpers ---
+    def _terminate_broker_atexit(self) -> None:
+        """Atexit-safe broker termination.
+
+        Avoids raising exceptions during interpreter shutdown.
+        """
+        try:
+            self._terminate_broker()
+        except Exception:
+            # Swallow errors at atexit to avoid noisy shutdowns
+            pass
+
+    def _terminate_broker(self) -> None:
+        """Terminate the SimpleMessageBroker process if running."""
+        proc = self._broker_process
+        if not proc:
+            return
+        try:
+            if hasattr(proc, "is_alive") and not proc.is_alive():
+                return
+        except Exception:
+            # If querying state fails, continue with termination attempt
+            pass
+
+        pid = getattr(proc, "pid", None)
+        logger.info(f"Stopping SimpleMessageBroker (pid={pid})")
+        try:
+            # First, try graceful terminate
+            proc.terminate()
+            try:
+                proc.join(timeout=3.0)
+            except Exception:
+                pass
+
+            # If still alive, escalate to SIGKILL on the single process
+            still_alive = False
+            try:
+                still_alive = hasattr(proc, "is_alive") and proc.is_alive()
+            except Exception:
+                still_alive = True
+            if still_alive and pid is not None:
+                try:
+                    os.kill(pid, signal.SIGKILL)
+                except Exception:
+                    pass
+                try:
+                    proc.join(timeout=2.0)
+                except Exception:
+                    pass
+        finally:
+            # Clear handle to avoid repeated attempts
+            self._broker_process = None

nv_ingest/framework/orchestration/process/strategies.py

@@ -0,0 +1,218 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Process execution strategies for pipeline deployment.
+
+This module defines abstract and concrete strategies for executing pipelines
+in different process contexts (in-process vs subprocess), implementing the
+Strategy pattern for clean separation of execution concerns.
+"""
+
+import atexit
+import logging
+import multiprocessing
+import os
+import sys
+import time
+from abc import ABC, abstractmethod
+
+from nv_ingest.pipeline.pipeline_schema import PipelineConfigSchema
+from nv_ingest.framework.orchestration.execution.options import ExecutionOptions, ExecutionResult
+from nv_ingest.framework.orchestration.ray.primitives.ray_pipeline import (
+    RayPipelineInterface,
+    RayPipelineSubprocessInterface,
+)
+from nv_ingest.framework.orchestration.process.execution import (
+    launch_pipeline,
+    run_pipeline_process,
+)
+from nv_ingest.framework.orchestration.process.termination import (
+    kill_pipeline_process_group,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ProcessExecutionStrategy(ABC):
+    """
+    Abstract base class for pipeline execution strategies.
+
+    This class defines the interface for different ways of executing
+    a pipeline (in-process, subprocess, etc.) using the Strategy pattern.
+    """
+
+    @abstractmethod
+    def execute(self, config: PipelineConfigSchema, options: ExecutionOptions) -> ExecutionResult:
+        """
+        Execute a pipeline using this strategy.
+
+        Parameters
+        ----------
+        config : PipelineConfigSchema
+            Validated pipeline configuration to execute.
+        options : ExecutionOptions
+            Execution options controlling blocking behavior and output redirection.
+
+        Returns
+        -------
+        ExecutionResult
+            Result containing pipeline interface and/or timing information.
+        """
+        pass
+
+
+class InProcessStrategy(ProcessExecutionStrategy):
+    """
+    Strategy for executing pipelines in the current process.
+
+    This strategy runs the pipeline directly in the current Python process,
+    providing the most direct execution path with minimal overhead.
+    """
+
+    def execute(self, config: PipelineConfigSchema, options: ExecutionOptions) -> ExecutionResult:
+        """
+        Execute pipeline in the current process.
+
+        Parameters
+        ----------
+        config : PipelineConfigSchema
+            Pipeline configuration to execute.
+        options : ExecutionOptions
+            Execution options. stdout/stderr are ignored for in-process execution.
+
+        Returns
+        -------
+        ExecutionResult
+            Result with pipeline interface (non-blocking) or elapsed time (blocking).
+        """
+        logger.info("Executing pipeline in current process")
+
+        # Execute the pipeline using existing launch_pipeline function
+        # launch_pipeline returns raw RayPipeline object (not wrapped in interface)
+        pipeline, total_elapsed = launch_pipeline(
+            config,
+            block=options.block,
+            disable_dynamic_scaling=None,  # Already applied in config
+        )
+
+        if options.block:
+            logger.debug(f"Pipeline execution completed successfully in {total_elapsed:.2f} seconds.")
+            return ExecutionResult(interface=None, elapsed_time=total_elapsed)
+        else:
+            # Wrap the raw RayPipeline in RayPipelineInterface
+            interface = RayPipelineInterface(pipeline)
+            return ExecutionResult(interface=interface, elapsed_time=None)
+
+
+class SubprocessStrategy(ProcessExecutionStrategy):
+    """
+    Strategy for executing pipelines in a separate subprocess.
+
+    This strategy launches the pipeline in a separate Python process using
+    multiprocessing, providing process isolation and output redirection.
+    """
+
+    def execute(self, config: PipelineConfigSchema, options: ExecutionOptions) -> ExecutionResult:
+        """
+        Execute pipeline in a separate subprocess.
+
+        Parameters
+        ----------
+        config : PipelineConfigSchema
+            Pipeline configuration to execute.
+        options : ExecutionOptions
+            Execution options including output redirection streams.
+
+        Returns
+        -------
+        ExecutionResult
+            Result with subprocess interface (non-blocking) or elapsed time (blocking).
+        """
+        logger.info("Launching pipeline in Python subprocess using multiprocessing.")
+
+        # Create subprocess using fork context
+        start_method = "fork"
+        if sys.platform.lower() == "darwin":
+            start_method = "spawn"
+        ctx = multiprocessing.get_context(start_method)
+        process = ctx.Process(
+            target=run_pipeline_process,
+            args=(
+                config,
+                options.stdout,  # raw_stdout
+                options.stderr,  # raw_stderr
+            ),
+            daemon=False,
+        )
+
+        # Hint to the lifecycle manager to skip starting the broker in the parent
+        prev_val = os.environ.get("NV_INGEST_BROKER_IN_SUBPROCESS")
+        os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"] = "1"
+        try:
+            process.start()
+        finally:
+            # Restore original env to avoid affecting other code paths
+            if prev_val is None:
+                try:
+                    del os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"]
+                except KeyError:
+                    pass
+            else:
+                os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"] = prev_val
+        interface = RayPipelineSubprocessInterface(process)
+
+        if options.block:
+            # Block until subprocess completes, handling Ctrl+C to ensure teardown
+            start_time = time.time()
+            logger.info("Waiting for subprocess pipeline to complete...")
+            try:
+                process.join()
+            except KeyboardInterrupt:
+                logger.info("KeyboardInterrupt in parent; terminating subprocess group...")
+                try:
+                    pid = int(process.pid)
+                    kill_pipeline_process_group(pid)
+                finally:
+                    # Best-effort wait for process to exit
+                    try:
+                        process.join(timeout=5.0)
+                    except Exception:
+                        pass
+            finally:
+                logger.info("Pipeline subprocess completed or terminated.")
+            elapsed_time = time.time() - start_time
+            # If process ended with failure, surface it
+            if hasattr(process, "exitcode") and process.exitcode not in (0, None):
+                raise RuntimeError(f"Pipeline subprocess exited with code {process.exitcode}")
+            return ExecutionResult(interface=None, elapsed_time=elapsed_time)
+        else:
+            # Return interface for non-blocking execution
+            logger.info(f"Pipeline subprocess started (PID={process.pid})")
+            # Ensure we pass the Process object, not just the PID, to avoid AttributeError
+            # kill_pipeline_process_group expects a multiprocessing.Process instance
+            # Capture raw PID to avoid using multiprocessing APIs during interpreter shutdown
+            pid = int(process.pid)
+            atexit.register(kill_pipeline_process_group, pid)
+            return ExecutionResult(interface=interface, elapsed_time=None)
+
+
+def create_execution_strategy(run_in_subprocess: bool) -> ProcessExecutionStrategy:
+    """
+    Factory function to create the appropriate execution strategy.
+
+    Parameters
+    ----------
+    run_in_subprocess : bool
+        If True, creates SubprocessStrategy. If False, creates InProcessStrategy.
+
+    Returns
+    -------
+    ProcessExecutionStrategy
+        Configured execution strategy instance.
+    """
+    if run_in_subprocess:
+        return SubprocessStrategy()
+    else:
+        return InProcessStrategy()

nv_ingest/framework/orchestration/process/termination.py

@@ -0,0 +1,147 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Process termination utilities, isolated to avoid circular imports.
+
+This module provides functions to terminate a process and its entire process
+group safely, without depending on pipeline construction or Ray types.
+"""
+
+import logging
+import os
+import signal
+import time
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+def _safe_log(level: int, msg: str) -> None:
+    """Best-effort logging that won't emit handler tracebacks on closed streams.
+
+    Temporarily disables logging.raiseExceptions to prevent the logging module
+    from printing "--- Logging error ---" to stderr if a handler's stream is
+    already closed (common during process teardown). Falls back to writing to
+    sys.__stderr__ if available.
+    """
+    try:
+        import logging as _logging
+
+        prev = getattr(_logging, "raiseExceptions", True)
+        # Suppress handler errors being printed to stderr
+        _logging.raiseExceptions = False
+
+        # If there are no handlers, skip and use stderr fallback
+        if logger.handlers:
+            logger.log(level, msg)
+            return
+    except Exception:
+        # Intentionally ignore and try stderr fallback
+        pass
+    finally:
+        try:
+            import logging as _logging  # re-import safe even if earlier failed
+
+            _logging.raiseExceptions = prev  # type: ignore[name-defined]
+        except Exception:
+            pass
+
+    # Fallback to stderr if available
+    try:
+        import sys
+
+        if hasattr(sys, "__stderr__") and sys.__stderr__:
+            sys.__stderr__.write(msg + "\n")
+            sys.__stderr__.flush()
+    except Exception:
+        pass
+
+
+def kill_pipeline_process_group(process) -> None:
+    """
+    Kill a process and its entire process group.
+
+    Accepts either a multiprocessing.Process-like object exposing a ``pid`` attribute
+    or a raw PID integer. Sends SIGTERM to the process group first, and escalates
+    to SIGKILL if it does not terminate within a short grace period.
+
+    Parameters
+    ----------
+    process : multiprocessing.Process | int
+        Process handle (or a raw PID int) for the process whose process group should be terminated.
+    """
+    proc: Optional[object] = None
+    pid: Optional[int] = None
+
+    if isinstance(process, int):
+        pid = process
+    elif hasattr(process, "pid"):
+        proc = process
+        try:
+            pid = int(getattr(proc, "pid"))
+        except Exception as e:
+            raise AttributeError(f"Invalid process-like object without usable pid: {e}")
+    else:
+        raise AttributeError(
+            "kill_pipeline_process_group expects a multiprocessing.Process or a PID int (process-like object with .pid)"
+        )
+
+    if proc is not None and hasattr(proc, "is_alive") and not proc.is_alive():
+        _safe_log(logging.DEBUG, "Process already terminated")
+        return
+
+    if pid is None:
+        raise AttributeError("Unable to determine PID for process group termination")
+
+    _safe_log(logging.INFO, f"Terminating pipeline process group (PID: {pid})")
+
+    try:
+        # Send graceful termination to the entire process group
+        try:
+            pgid = os.getpgid(pid)
+        except Exception:
+            # Process already gone
+            _safe_log(logging.DEBUG, f"Process group for PID {pid} not found during SIGTERM phase")
+            return
+        try:
+            os.killpg(pgid, signal.SIGTERM)
+        except ProcessLookupError:
+            _safe_log(logging.DEBUG, f"Process group for PID {pid} no longer exists (SIGTERM)")
+            return
+
+        # If we have a Process handle, give it a chance to exit cleanly
+        if proc is not None and hasattr(proc, "join"):
+            try:
+                proc.join(timeout=5.0)
+            except Exception:
+                pass
+            still_alive = getattr(proc, "is_alive", lambda: True)()
+        else:
+            # Without a handle, provide a small grace period
+            time.sleep(2.0)
+            try:
+                _ = os.getpgid(pid)
+                still_alive = True
+            except Exception:
+                still_alive = False
+
+        if still_alive:
+            _safe_log(logging.WARNING, "Process group did not terminate gracefully, using SIGKILL")
+            try:
+                try:
+                    pgid2 = os.getpgid(pid)
+                except Exception:
+                    _safe_log(logging.DEBUG, f"Process group for PID {pid} vanished before SIGKILL")
+                    return
+                os.killpg(pgid2, signal.SIGKILL)
+            finally:
+                if proc is not None and hasattr(proc, "join"):
+                    try:
+                        proc.join(timeout=3.0)
+                    except Exception:
+                        pass
+
+    except (ProcessLookupError, OSError) as e:
+        _safe_log(logging.DEBUG, f"Process group already terminated or not found: {e}")

nv_ingest/framework/orchestration/ray/examples/pipeline_test_harness.py

@@ -29,8 +29,8 @@ from nv_ingest.framework.orchestration.ray.stages.sinks.message_broker_task_sink
 from nv_ingest.framework.orchestration.ray.stages.sources.message_broker_task_source import (
     MessageBrokerTaskSourceStage,
     MessageBrokerTaskSourceConfig,
-    start_simple_message_broker,
 )
+from nv_ingest.framework.orchestration.process.dependent_services import start_simple_message_broker
 from nv_ingest.framework.orchestration.ray.stages.storage.image_storage import ImageStorageStage
 from nv_ingest.framework.orchestration.ray.stages.storage.store_embeddings import EmbeddingStorageStage
 from nv_ingest.framework.orchestration.ray.stages.transforms.image_caption import ImageCaptionTransformStage
@@ -152,11 +152,11 @@ if __name__ == "__main__":
     os.environ["OCR_MODEL_NAME"] = "paddle"
     os.environ["NEMORETRIEVER_PARSE_HTTP_ENDPOINT"] = "https://integrate.api.nvidia.com/v1/chat/completions"
     os.environ["VLM_CAPTION_ENDPOINT"] = "https://integrate.api.nvidia.com/v1/chat/completions"
-    os.environ["VLM_CAPTION_MODEL_NAME"] = "nvidia/llama-3.1-nemotron-nano-vl-8b-v1"
+    os.environ["VLM_CAPTION_MODEL_NAME"] = "nvidia/nemotron-nano-12b-v2-vl"
     logger.info("Environment variables set.")
 
     image_caption_endpoint_url = "https://integrate.api.nvidia.com/v1/chat/completions"
-    model_name = "nvidia/llama-3.1-nemotron-nano-vl-8b-v1"
+    model_name = "nvidia/nemotron-nano-12b-v2-vl"
     yolox_grpc, yolox_http, yolox_auth, yolox_protocol = get_nim_service("yolox")
     (
         yolox_table_structure_grpc,

nv_ingest/framework/orchestration/ray/primitives/pipeline_topology.py

@@ -183,7 +183,7 @@ class PipelineTopology:
         """Marks an actor as pending removal, to be cleaned up by the background thread."""
         with self._lock:
             self._actors_pending_removal.add((stage_name, actor))
-            logger.info(f"Marked actor {actor} from stage {stage_name} for removal.")
+            logger.debug(f"Marked actor {actor} from stage {stage_name} for removal.")
 
     def start_cleanup_thread(self, interval: int = 5) -> None:
         """Starts the background thread for periodic cleanup tasks."""
@@ -191,14 +191,14 @@ class PipelineTopology:
             self._stop_cleanup.clear()
             self._cleanup_thread = threading.Thread(target=self._cleanup_loop, args=(interval,), daemon=True)
             self._cleanup_thread.start()
-            logger.info("Topology cleanup thread started.")
+            logger.debug("Topology cleanup thread started.")
 
     def stop_cleanup_thread(self) -> None:
         """Stops the background cleanup thread."""
         if self._cleanup_thread and self._cleanup_thread.is_alive():
             self._stop_cleanup.set()
             self._cleanup_thread.join(timeout=5)
-            logger.info("Topology cleanup thread stopped.")
+            logger.debug("Topology cleanup thread stopped.")
 
     def _cleanup_loop(self, interval: int) -> None:
         """Periodically checks for and removes actors that have completed shutdown."""
@@ -235,7 +235,7 @@ class PipelineTopology:
                             self._actors_pending_removal.remove((stage_name, actor))
                         if actor in self._stage_actors.get(stage_name, []):
                             self._stage_actors[stage_name].remove(actor)
-                            logger.info(f"Successfully removed actor {actor} from stage {stage_name} in topology.")
+                            logger.debug(f"Successfully removed actor {actor} from stage {stage_name} in topology.")
 
             time.sleep(interval)