nv-ingest 2025.8.19.dev20250819__tar.gz → 2025.8.20.dev20250820__tar.gz

{nv_ingest-2025.8.19.dev20250819 → nv_ingest-2025.8.20.dev20250820}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.8.19.dev20250819
+Version: 2025.8.20.dev20250820
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

nv_ingest-2025.8.20.dev20250820/nv_ingest/framework/orchestration/process/dependent_services.py

@@ -0,0 +1,77 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Dependent services management for pipeline orchestration.
+
+This module contains utilities for starting and managing dependent services
+that the pipeline requires, such as message brokers and other infrastructure.
+"""
+
+import logging
+import os
+import multiprocessing
+import socket
+from nv_ingest_api.util.message_brokers.simple_message_broker.broker import SimpleMessageBroker
+
+logger = logging.getLogger(__name__)
+
+
+def start_simple_message_broker(broker_client: dict) -> multiprocessing.Process:
+    """
+    Starts a SimpleMessageBroker server in a separate process.
+
+    Parameters
+    ----------
+    broker_client : dict
+        Broker configuration. Expected keys include:
+          - "port": the port to bind the server to,
+          - "broker_params": optionally including "max_queue_size",
+          - and any other parameters required by SimpleMessageBroker.
+
+    Returns
+    -------
+    multiprocessing.Process
+        The process running the SimpleMessageBroker server.
+    """
+
+    # Resolve host/port early for pre-flight checks
+    broker_params = broker_client.get("broker_params", {})
+    max_queue_size = broker_params.get("max_queue_size", 10000)
+    server_host = broker_client.get("host", "0.0.0.0")
+    server_port = broker_client.get("port", 7671)
+
+    # Pre-flight: if something is already listening on the target port, do not spawn another broker.
+    # This avoids noisy stack traces from a failing child process when tests/pipeline are run repeatedly.
+    def _is_port_open(host: str, port: int) -> bool:
+        check_host = "127.0.0.1" if host in ("0.0.0.0", "::") else host
+        try:
+            with socket.create_connection((check_host, port), timeout=0.5):
+                return True
+        except Exception:
+            return False
+
+    if _is_port_open(server_host, server_port):
+        logger.warning(
+            f"SimpleMessageBroker port already in use at {server_host}:{server_port}; "
+            f"continuing to spawn a broker process (tests expect a Process to be returned)"
+        )
+
+    def broker_server():
+        # Optionally, set socket options here for reuse (note: binding occurs in server __init__).
+        server = SimpleMessageBroker(server_host, server_port, max_queue_size)
+        try:
+            server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        except Exception:
+            pass
+        server.serve_forever()
+
+    p = multiprocessing.Process(target=broker_server)
+    # If we're launching from inside the pipeline subprocess, mark daemon so the
+    # broker dies automatically when the subprocess exits.
+    p.daemon = os.environ.get("NV_INGEST_BROKER_IN_SUBPROCESS") == "1"
+    p.start()
+    logger.info(f"Started SimpleMessageBroker server in separate process on port {server_port}")
+
+    return p

{nv_ingest-2025.8.19.dev20250819 → nv_ingest-2025.8.20.dev20250820}/nv_ingest/framework/orchestration/process/execution.py

@@ -17,14 +17,15 @@ import sys
 import time
 from ctypes import CDLL
 from datetime import datetime
-from typing import Union, Tuple, Optional, TextIO
+from typing import Union, Tuple, Optional, TextIO, Any
 import json
 
 import ray
 from ray import LoggingConfig
 
-from nv_ingest.framework.orchestration.ray.primitives.ray_pipeline import (
-    RayPipeline,
+from nv_ingest.framework.orchestration.process.dependent_services import start_simple_message_broker
+from nv_ingest.framework.orchestration.process.termination import (
+    kill_pipeline_process_group as _kill_pipeline_process_group,
 )
 from nv_ingest.pipeline.ingest_pipeline import IngestPipelineBuilder
 from nv_ingest.pipeline.pipeline_schema import PipelineConfigSchema
@@ -250,7 +251,7 @@ def launch_pipeline(
     block: bool = True,
     disable_dynamic_scaling: Optional[bool] = None,
     dynamic_memory_threshold: Optional[float] = None,
-) -> Tuple[Union[RayPipeline, None], Optional[float]]:
+) -> Tuple[Union[Any, None], Optional[float]]:
     """
     Launch a pipeline using the provided configuration.
 
@@ -270,8 +271,8 @@ def launch_pipeline(
 
     Returns
     -------
-    Tuple[Union[RayPipeline, None], Optional[float]]
-        Raw RayPipeline object and elapsed time. For blocking execution,
+    Tuple[Union[Any, None], Optional[float]]
+        Raw pipeline object (type elided to avoid circular import) and elapsed time. For blocking execution,
         returns (None, elapsed_time). For non-blocking, returns (pipeline, None).
     """
     logger.info("Starting pipeline setup")
@@ -328,17 +329,37 @@ def launch_pipeline(
 
     # Set up the ingestion pipeline
     start_abs = datetime.now()
-    ingest_pipeline = IngestPipelineBuilder(pipeline_config)
-    ingest_pipeline.build()
+    ingest_pipeline = None
+    try:
+        ingest_pipeline = IngestPipelineBuilder(pipeline_config)
+        ingest_pipeline.build()
 
-    # Record setup time
-    end_setup = start_run = datetime.now()
-    setup_time = (end_setup - start_abs).total_seconds()
-    logger.info(f"Pipeline setup complete in {setup_time:.2f} seconds")
+        # Record setup time
+        end_setup = start_run = datetime.now()
+        setup_time = (end_setup - start_abs).total_seconds()
+        logger.info(f"Pipeline setup complete in {setup_time:.2f} seconds")
 
-    # Run the pipeline
-    logger.debug("Running pipeline")
-    ingest_pipeline.start()
+        # Run the pipeline
+        logger.debug("Running pipeline")
+        ingest_pipeline.start()
+    except Exception as e:
+        # Ensure any partial startup is torn down
+        logger.error(f"Pipeline startup failed, initiating cleanup: {e}", exc_info=True)
+        try:
+            if ingest_pipeline is not None:
+                try:
+                    ingest_pipeline.stop()
+                except Exception:
+                    pass
+        finally:
+            try:
+                if ray.is_initialized():
+                    ray.shutdown()
+                    logger.info("Ray shutdown complete after startup failure.")
+            finally:
+                pass
+        # Re-raise to surface failure to caller
+        raise
 
     if block:
         try:
@@ -350,6 +371,14 @@ def launch_pipeline(
             ingest_pipeline.stop()
             ray.shutdown()
             logger.info("Ray shutdown complete.")
+        except Exception as e:
+            logger.error(f"Unexpected error during pipeline run: {e}", exc_info=True)
+            try:
+                ingest_pipeline.stop()
+            finally:
+                if ray.is_initialized():
+                    ray.shutdown()
+            raise
 
         # Record execution times
         end_run = datetime.now()
@@ -392,12 +421,34 @@ def run_pipeline_process(
     if stderr:
         sys.stderr = stderr
 
+    # Ensure the subprocess is killed if the parent dies to avoid hangs
+    try:
+        set_pdeathsig(signal.SIGKILL)
+    except Exception as e:
+        logger.debug(f"set_pdeathsig not available or failed: {e}")
+
     # Create a new process group so we can terminate the entire subtree cleanly
     try:
         os.setpgrp()
     except Exception as e:
         logger.debug(f"os.setpgrp() not available or failed: {e}")
 
+    # Install signal handlers for graceful shutdown in the subprocess
+    def _handle_signal(signum, frame):
+        try:
+            _safe_log(logging.INFO, f"Received signal {signum}; shutting down Ray and exiting...")
+            if ray.is_initialized():
+                ray.shutdown()
+        finally:
+            # Exit immediately after best-effort cleanup
+            os._exit(0)
+
+    try:
+        signal.signal(signal.SIGINT, _handle_signal)
+        signal.signal(signal.SIGTERM, _handle_signal)
+    except Exception as e:
+        logger.debug(f"Signal handlers not set: {e}")
+
     # Test output redirection
     print("DEBUG: Direct print to stdout - should appear in parent process")
     sys.stderr.write("DEBUG: Direct write to stderr - should appear in parent process\n")
@@ -405,93 +456,40 @@ def run_pipeline_process(
     # Test logging output
     logger.info("DEBUG: Logger info - may not appear if logging handlers not redirected")
 
+    # If requested, start the simple broker inside this subprocess so it shares the process group
+    broker_proc = None
     try:
+        if os.environ.get("NV_INGEST_BROKER_IN_SUBPROCESS") == "1":
+            try:
+                # Only launch if the config requests it
+                if getattr(pipeline_config, "pipeline", None) and getattr(
+                    pipeline_config.pipeline, "launch_simple_broker", False
+                ):
+                    _safe_log(logging.INFO, "Starting SimpleMessageBroker inside subprocess")
+                    broker_proc = start_simple_message_broker({})
+            except Exception as e:
+                _safe_log(logging.ERROR, f"Failed to start SimpleMessageBroker in subprocess: {e}")
+                # Continue without broker; launch will fail fast if required
+
         # Launch the pipeline (blocking)
         launch_pipeline(pipeline_config, block=True)
 
     except Exception as e:
         logger.error(f"Subprocess pipeline execution failed: {e}")
         raise
-
-
-def kill_pipeline_process_group(process: multiprocessing.Process) -> None:
-    """
-    Kill a pipeline process and its entire process group.
-
-    Note: Although the type annotation specifies a multiprocessing.Process for
-    compatibility with existing tests and public API, this function is robust
-    to also being passed a raw PID (int) at runtime.
-
-    Behavior:
-    - Send SIGTERM to the process group; if still alive after grace period, escalate to SIGKILL.
-    - If a Process object is provided, attempt to join() with timeouts.
-    - If only a PID is provided, skip joins and just signal the process group with grace/force.
-
-    Parameters
-    ----------
-    process : multiprocessing.Process
-        Process handle (or a raw PID int) for the process whose process group should be terminated.
-    """
-    # Resolve PID and optional Process handle
-    proc: Optional[object] = None
-    pid: Optional[int] = None
-
-    if isinstance(process, int):
-        pid = process
-    elif hasattr(process, "pid"):
-        # Duck-type any object that exposes a pid (e.g., multiprocessing.Process or Mock)
-        proc = process
-        try:
-            pid = int(getattr(proc, "pid"))
-        except Exception as e:
-            raise AttributeError(f"Invalid process-like object without usable pid: {e}")
-    else:
-        raise AttributeError(
-            "kill_pipeline_process_group expects a multiprocessing.Process or a PID int (process-like object with .pid)"
-        )
-
-    # If we have a Process handle and it's already dead, nothing to do
-    if proc is not None and hasattr(proc, "is_alive") and not proc.is_alive():
-        _safe_log(logging.DEBUG, "Process already terminated")
-        return
-
-    if pid is None:
-        # Defensive guard; should not happen
-        raise AttributeError("Unable to determine PID for process group termination")
-
-    _safe_log(logging.INFO, f"Terminating pipeline process group (PID: {pid})")
-    try:
-        # Send graceful termination to the entire process group
-        os.killpg(os.getpgid(pid), signal.SIGTERM)
-
-        # If we have a Process handle, give it a chance to exit cleanly
-        if proc is not None and hasattr(proc, "join"):
+    finally:
+        # Best-effort: if we created a broker here and the pipeline exits normally,
+        # attempt a graceful terminate. In failure/termination paths the process group kill
+        # from parent or signal handler will take care of it.
+        if broker_proc is not None:
             try:
-                proc.join(timeout=5.0)
+                if hasattr(broker_proc, "is_alive") and broker_proc.is_alive():
+                    broker_proc.terminate()
             except Exception:
                 pass
-            still_alive = getattr(proc, "is_alive", lambda: True)()
-        else:
-            # Without a handle, provide a small grace period
-            time.sleep(2.0)
-            # Best-effort check: if getpgid fails, it's gone
-            try:
-                _ = os.getpgid(pid)
-                still_alive = True
-            except Exception:
-                still_alive = False
 
-        if still_alive:
-            _safe_log(logging.WARNING, "Process group did not terminate gracefully, using SIGKILL")
-            try:
-                os.killpg(os.getpgid(pid), signal.SIGKILL)
-            finally:
-                if proc is not None and hasattr(proc, "join"):
-                    try:
-                        proc.join(timeout=3.0)
-                    except Exception:
-                        pass
-
-    except (ProcessLookupError, OSError) as e:
-        # Process or group may already be gone
-        _safe_log(logging.DEBUG, f"Process group already terminated or not found: {e}")
+
+def kill_pipeline_process_group(process: multiprocessing.Process) -> None:
+    """Backward-compatible shim that delegates to process.termination implementation."""
+    _safe_log(logging.DEBUG, "Delegating kill_pipeline_process_group to process.termination module")
+    _kill_pipeline_process_group(process)

nv_ingest-2025.8.20.dev20250820/nv_ingest/framework/orchestration/process/lifecycle.py

@@ -0,0 +1,214 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Pipeline lifecycle management for declarative execution.
+
+This module provides high-level lifecycle management for pipelines,
+orchestrating configuration resolution, broker setup, and execution
+using the configured strategy pattern.
+"""
+
+import logging
+import atexit
+import multiprocessing
+import os
+import signal
+from typing import Optional
+
+from nv_ingest.pipeline.pipeline_schema import PipelineConfigSchema
+from nv_ingest.framework.orchestration.execution.options import ExecutionOptions, ExecutionResult
+from nv_ingest.framework.orchestration.process.strategies import ProcessExecutionStrategy
+from nv_ingest.framework.orchestration.process.strategies import SubprocessStrategy
+from nv_ingest.framework.orchestration.process.dependent_services import start_simple_message_broker
+
+logger = logging.getLogger(__name__)
+
+
+class PipelineLifecycleManager:
+    """
+    High-level manager for pipeline lifecycle operations.
+
+    This class orchestrates the complete pipeline lifecycle including
+    broker setup, configuration validation, and execution using the
+    configured execution strategy.
+
+    Attributes
+    ----------
+    strategy : ProcessExecutionStrategy
+        The execution strategy to use for running pipelines.
+    """
+
+    def __init__(self, strategy: ProcessExecutionStrategy):
+        """
+        Initialize the lifecycle manager with an execution strategy.
+
+        Parameters
+        ----------
+        strategy : ProcessExecutionStrategy
+            The strategy to use for pipeline execution.
+        """
+        self.strategy = strategy
+        # Track broker process so we can terminate it during teardown
+        self._broker_process: Optional[multiprocessing.Process] = None
+
+    def start(self, config: PipelineConfigSchema, options: ExecutionOptions) -> ExecutionResult:
+        """
+        Start a pipeline using the configured execution strategy.
+
+        This method handles the complete pipeline startup process:
+        1. Validate configuration
+        2. Start message broker if required
+        3. Execute pipeline using the configured strategy
+
+        Parameters
+        ----------
+        config : PipelineConfigSchema
+            Validated pipeline configuration to execute.
+        options : ExecutionOptions
+            Execution options controlling blocking behavior and output.
+
+        Returns
+        -------
+        ExecutionResult
+            Result containing pipeline interface and/or timing information.
+
+        Raises
+        ------
+        RuntimeError
+            If pipeline startup fails.
+        """
+        logger.info("Starting pipeline lifecycle")
+
+        # If running pipeline in a subprocess and broker is enabled, ensure the broker
+        # is launched in the child process group by signaling via environment variable
+        prev_env = None
+        set_env = False
+        if getattr(config, "pipeline", None) and getattr(config.pipeline, "launch_simple_broker", False):
+            if isinstance(self.strategy, SubprocessStrategy):
+                prev_env = os.environ.get("NV_INGEST_BROKER_IN_SUBPROCESS")
+                os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"] = "1"
+                set_env = True
+
+        try:
+            # Start message broker if configured (may defer to subprocess based on env)
+            self._setup_message_broker(config)
+
+            # Execute pipeline using the configured strategy
+            result = self.strategy.execute(config, options)
+
+            logger.info("Pipeline lifecycle started successfully")
+            return result
+
+        except Exception as e:
+            logger.error(f"Failed to start pipeline lifecycle: {e}")
+            raise RuntimeError(f"Pipeline startup failed: {e}") from e
+        finally:
+            if set_env:
+                if prev_env is None:
+                    try:
+                        del os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"]
+                    except KeyError:
+                        pass
+                else:
+                    os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"] = prev_env
+
+    def _setup_message_broker(self, config: PipelineConfigSchema) -> None:
+        """
+        Set up message broker if required by configuration.
+
+        Parameters
+        ----------
+        config : PipelineConfigSchema
+            Pipeline configuration containing broker settings.
+        """
+        if config.pipeline.launch_simple_broker:
+            # If requested to launch broker inside the subprocess, skip here
+            if os.environ.get("NV_INGEST_BROKER_IN_SUBPROCESS") == "1":
+                logger.info("Deferring SimpleMessageBroker launch to subprocess")
+                return
+            logger.info("Starting simple message broker")
+            # Start the broker and retain a handle for cleanup.
+            # Use defaults (host=0.0.0.0, port=7671) as set by the broker implementation.
+            try:
+                self._broker_process = start_simple_message_broker({})
+                # Ensure cleanup at interpreter shutdown in case caller forgets
+                atexit.register(self._terminate_broker_atexit)
+                logger.info(f"SimpleMessageBroker started (pid={getattr(self._broker_process, 'pid', None)})")
+            except Exception as e:
+                logger.error(f"Failed to start SimpleMessageBroker: {e}")
+                raise
+        else:
+            logger.debug("Simple broker launch not required")
+
+    def stop(self, pipeline_id: Optional[str] = None) -> None:
+        """
+        Stop a running pipeline.
+
+        This method provides a hook for future pipeline stopping functionality.
+        Currently, pipeline stopping is handled by the individual interfaces.
+        Additionally, it ensures any dependent services (like the simple
+        message broker) are terminated to avoid lingering processes.
+
+        Parameters
+        ----------
+        pipeline_id : Optional[str]
+            Identifier of the pipeline to stop. Currently unused.
+        """
+        logger.info("Pipeline stop requested")
+        # Best-effort termination of broker if we started one
+        self._terminate_broker()
+
+    # --- Internal helpers ---
+    def _terminate_broker_atexit(self) -> None:
+        """Atexit-safe broker termination.
+
+        Avoids raising exceptions during interpreter shutdown.
+        """
+        try:
+            self._terminate_broker()
+        except Exception:
+            # Swallow errors at atexit to avoid noisy shutdowns
+            pass
+
+    def _terminate_broker(self) -> None:
+        """Terminate the SimpleMessageBroker process if running."""
+        proc = self._broker_process
+        if not proc:
+            return
+        try:
+            if hasattr(proc, "is_alive") and not proc.is_alive():
+                return
+        except Exception:
+            # If querying state fails, continue with termination attempt
+            pass
+
+        pid = getattr(proc, "pid", None)
+        logger.info(f"Stopping SimpleMessageBroker (pid={pid})")
+        try:
+            # First, try graceful terminate
+            proc.terminate()
+            try:
+                proc.join(timeout=3.0)
+            except Exception:
+                pass
+
+            # If still alive, escalate to SIGKILL on the single process
+            still_alive = False
+            try:
+                still_alive = hasattr(proc, "is_alive") and proc.is_alive()
+            except Exception:
+                still_alive = True
+            if still_alive and pid is not None:
+                try:
+                    os.kill(pid, signal.SIGKILL)
+                except Exception:
+                    pass
+                try:
+                    proc.join(timeout=2.0)
+                except Exception:
+                    pass
+        finally:
+            # Clear handle to avoid repeated attempts
+            self._broker_process = None

{nv_ingest-2025.8.19.dev20250819 → nv_ingest-2025.8.20.dev20250820}/nv_ingest/framework/orchestration/process/strategies.py

@@ -11,6 +11,7 @@ Strategy pattern for clean separation of execution concerns.
 """
 
 import atexit
+import os
 import logging
 import multiprocessing
 import time
@@ -25,6 +26,8 @@ from nv_ingest.framework.orchestration.ray.primitives.ray_pipeline import (
 from nv_ingest.framework.orchestration.process.execution import (
     launch_pipeline,
     run_pipeline_process,
+)
+from nv_ingest.framework.orchestration.process.termination import (
     kill_pipeline_process_group,
 )
 
@@ -140,16 +143,45 @@ class SubprocessStrategy(ProcessExecutionStrategy):
             daemon=False,
         )
 
-        process.start()
+        # Hint to the lifecycle manager to skip starting the broker in the parent
+        prev_val = os.environ.get("NV_INGEST_BROKER_IN_SUBPROCESS")
+        os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"] = "1"
+        try:
+            process.start()
+        finally:
+            # Restore original env to avoid affecting other code paths
+            if prev_val is None:
+                try:
+                    del os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"]
+                except KeyError:
+                    pass
+            else:
+                os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"] = prev_val
         interface = RayPipelineSubprocessInterface(process)
 
         if options.block:
-            # Block until subprocess completes
+            # Block until subprocess completes, handling Ctrl+C to ensure teardown
             start_time = time.time()
             logger.info("Waiting for subprocess pipeline to complete...")
-            process.join()
-            logger.info("Pipeline subprocess completed.")
+            try:
+                process.join()
+            except KeyboardInterrupt:
+                logger.info("KeyboardInterrupt in parent; terminating subprocess group...")
+                try:
+                    pid = int(process.pid)
+                    kill_pipeline_process_group(pid)
+                finally:
+                    # Best-effort wait for process to exit
+                    try:
+                        process.join(timeout=5.0)
+                    except Exception:
+                        pass
+            finally:
+                logger.info("Pipeline subprocess completed or terminated.")
             elapsed_time = time.time() - start_time
+            # If process ended with failure, surface it
+            if hasattr(process, "exitcode") and process.exitcode not in (0, None):
+                raise RuntimeError(f"Pipeline subprocess exited with code {process.exitcode}")
             return ExecutionResult(interface=None, elapsed_time=elapsed_time)
         else:
             # Return interface for non-blocking execution