nv-ingest 2025.8.18.dev20250818__py3-none-any.whl → 2025.8.20.dev20250820__py3-none-any.whl

nv_ingest/framework/orchestration/process/dependent_services.py

@@ -10,6 +10,7 @@ that the pipeline requires, such as message brokers and other infrastructure.
 """
 
 import logging
+import os
 import multiprocessing
 import socket
 from nv_ingest_api.util.message_brokers.simple_message_broker.broker import SimpleMessageBroker
@@ -35,21 +36,42 @@ def start_simple_message_broker(broker_client: dict) -> multiprocessing.Process:
         The process running the SimpleMessageBroker server.
     """
 
+    # Resolve host/port early for pre-flight checks
+    broker_params = broker_client.get("broker_params", {})
+    max_queue_size = broker_params.get("max_queue_size", 10000)
+    server_host = broker_client.get("host", "0.0.0.0")
+    server_port = broker_client.get("port", 7671)
+
+    # Pre-flight: if something is already listening on the target port, do not spawn another broker.
+    # This avoids noisy stack traces from a failing child process when tests/pipeline are run repeatedly.
+    def _is_port_open(host: str, port: int) -> bool:
+        check_host = "127.0.0.1" if host in ("0.0.0.0", "::") else host
+        try:
+            with socket.create_connection((check_host, port), timeout=0.5):
+                return True
+        except Exception:
+            return False
+
+    if _is_port_open(server_host, server_port):
+        logger.warning(
+            f"SimpleMessageBroker port already in use at {server_host}:{server_port}; "
+            f"continuing to spawn a broker process (tests expect a Process to be returned)"
+        )
+
     def broker_server():
-        # Use max_queue_size from broker_params or default to 10000.
-        broker_params = broker_client.get("broker_params", {})
-        max_queue_size = broker_params.get("max_queue_size", 10000)
-        server_host = broker_client.get("host", "0.0.0.0")
-        server_port = broker_client.get("port", 7671)
-        # Optionally, set socket options here for reuse.
+        # Optionally, set socket options here for reuse (note: binding occurs in server __init__).
         server = SimpleMessageBroker(server_host, server_port, max_queue_size)
-        # Enable address reuse on the server socket.
-        server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        try:
+            server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        except Exception:
+            pass
         server.serve_forever()
 
     p = multiprocessing.Process(target=broker_server)
-    p.daemon = False
+    # If we're launching from inside the pipeline subprocess, mark daemon so the
+    # broker dies automatically when the subprocess exits.
+    p.daemon = os.environ.get("NV_INGEST_BROKER_IN_SUBPROCESS") == "1"
     p.start()
-    logger.info(f"Started SimpleMessageBroker server in separate process on port {broker_client.get('port', 7671)}")
+    logger.info(f"Started SimpleMessageBroker server in separate process on port {server_port}")
 
     return p

nv_ingest/framework/orchestration/process/execution.py

@@ -17,14 +17,15 @@ import sys
 import time
 from ctypes import CDLL
 from datetime import datetime
-from typing import Union, Tuple, Optional, TextIO
+from typing import Union, Tuple, Optional, TextIO, Any
 import json
 
 import ray
 from ray import LoggingConfig
 
-from nv_ingest.framework.orchestration.ray.primitives.ray_pipeline import (
-    RayPipeline,
+from nv_ingest.framework.orchestration.process.dependent_services import start_simple_message_broker
+from nv_ingest.framework.orchestration.process.termination import (
+    kill_pipeline_process_group as _kill_pipeline_process_group,
 )
 from nv_ingest.pipeline.ingest_pipeline import IngestPipelineBuilder
 from nv_ingest.pipeline.pipeline_schema import PipelineConfigSchema
@@ -250,7 +251,7 @@ def launch_pipeline(
     block: bool = True,
     disable_dynamic_scaling: Optional[bool] = None,
     dynamic_memory_threshold: Optional[float] = None,
-) -> Tuple[Union[RayPipeline, None], Optional[float]]:
+) -> Tuple[Union[Any, None], Optional[float]]:
     """
     Launch a pipeline using the provided configuration.
 
@@ -270,8 +271,8 @@ def launch_pipeline(
 
     Returns
     -------
-    Tuple[Union[RayPipeline, None], Optional[float]]
-        Raw RayPipeline object and elapsed time. For blocking execution,
+    Tuple[Union[Any, None], Optional[float]]
+        Raw pipeline object (type elided to avoid circular import) and elapsed time. For blocking execution,
         returns (None, elapsed_time). For non-blocking, returns (pipeline, None).
     """
     logger.info("Starting pipeline setup")
@@ -328,17 +329,37 @@ def launch_pipeline(
 
     # Set up the ingestion pipeline
     start_abs = datetime.now()
-    ingest_pipeline = IngestPipelineBuilder(pipeline_config)
-    ingest_pipeline.build()
+    ingest_pipeline = None
+    try:
+        ingest_pipeline = IngestPipelineBuilder(pipeline_config)
+        ingest_pipeline.build()
 
-    # Record setup time
-    end_setup = start_run = datetime.now()
-    setup_time = (end_setup - start_abs).total_seconds()
-    logger.info(f"Pipeline setup complete in {setup_time:.2f} seconds")
+        # Record setup time
+        end_setup = start_run = datetime.now()
+        setup_time = (end_setup - start_abs).total_seconds()
+        logger.info(f"Pipeline setup complete in {setup_time:.2f} seconds")
 
-    # Run the pipeline
-    logger.debug("Running pipeline")
-    ingest_pipeline.start()
+        # Run the pipeline
+        logger.debug("Running pipeline")
+        ingest_pipeline.start()
+    except Exception as e:
+        # Ensure any partial startup is torn down
+        logger.error(f"Pipeline startup failed, initiating cleanup: {e}", exc_info=True)
+        try:
+            if ingest_pipeline is not None:
+                try:
+                    ingest_pipeline.stop()
+                except Exception:
+                    pass
+        finally:
+            try:
+                if ray.is_initialized():
+                    ray.shutdown()
+                    logger.info("Ray shutdown complete after startup failure.")
+            finally:
+                pass
+        # Re-raise to surface failure to caller
+        raise
 
     if block:
         try:
@@ -350,6 +371,14 @@ def launch_pipeline(
             ingest_pipeline.stop()
             ray.shutdown()
             logger.info("Ray shutdown complete.")
+        except Exception as e:
+            logger.error(f"Unexpected error during pipeline run: {e}", exc_info=True)
+            try:
+                ingest_pipeline.stop()
+            finally:
+                if ray.is_initialized():
+                    ray.shutdown()
+            raise
 
         # Record execution times
         end_run = datetime.now()
@@ -392,12 +421,34 @@ def run_pipeline_process(
     if stderr:
         sys.stderr = stderr
 
+    # Ensure the subprocess is killed if the parent dies to avoid hangs
+    try:
+        set_pdeathsig(signal.SIGKILL)
+    except Exception as e:
+        logger.debug(f"set_pdeathsig not available or failed: {e}")
+
     # Create a new process group so we can terminate the entire subtree cleanly
     try:
         os.setpgrp()
     except Exception as e:
         logger.debug(f"os.setpgrp() not available or failed: {e}")
 
+    # Install signal handlers for graceful shutdown in the subprocess
+    def _handle_signal(signum, frame):
+        try:
+            _safe_log(logging.INFO, f"Received signal {signum}; shutting down Ray and exiting...")
+            if ray.is_initialized():
+                ray.shutdown()
+        finally:
+            # Exit immediately after best-effort cleanup
+            os._exit(0)
+
+    try:
+        signal.signal(signal.SIGINT, _handle_signal)
+        signal.signal(signal.SIGTERM, _handle_signal)
+    except Exception as e:
+        logger.debug(f"Signal handlers not set: {e}")
+
     # Test output redirection
     print("DEBUG: Direct print to stdout - should appear in parent process")
     sys.stderr.write("DEBUG: Direct write to stderr - should appear in parent process\n")
@@ -405,93 +456,40 @@ def run_pipeline_process(
     # Test logging output
     logger.info("DEBUG: Logger info - may not appear if logging handlers not redirected")
 
+    # If requested, start the simple broker inside this subprocess so it shares the process group
+    broker_proc = None
     try:
+        if os.environ.get("NV_INGEST_BROKER_IN_SUBPROCESS") == "1":
+            try:
+                # Only launch if the config requests it
+                if getattr(pipeline_config, "pipeline", None) and getattr(
+                    pipeline_config.pipeline, "launch_simple_broker", False
+                ):
+                    _safe_log(logging.INFO, "Starting SimpleMessageBroker inside subprocess")
+                    broker_proc = start_simple_message_broker({})
+            except Exception as e:
+                _safe_log(logging.ERROR, f"Failed to start SimpleMessageBroker in subprocess: {e}")
+                # Continue without broker; launch will fail fast if required
+
         # Launch the pipeline (blocking)
         launch_pipeline(pipeline_config, block=True)
 
     except Exception as e:
         logger.error(f"Subprocess pipeline execution failed: {e}")
         raise
-
-
-def kill_pipeline_process_group(process: multiprocessing.Process) -> None:
-    """
-    Kill a pipeline process and its entire process group.
-
-    Note: Although the type annotation specifies a multiprocessing.Process for
-    compatibility with existing tests and public API, this function is robust
-    to also being passed a raw PID (int) at runtime.
-
-    Behavior:
-    - Send SIGTERM to the process group; if still alive after grace period, escalate to SIGKILL.
-    - If a Process object is provided, attempt to join() with timeouts.
-    - If only a PID is provided, skip joins and just signal the process group with grace/force.
-
-    Parameters
-    ----------
-    process : multiprocessing.Process
-        Process handle (or a raw PID int) for the process whose process group should be terminated.
-    """
-    # Resolve PID and optional Process handle
-    proc: Optional[object] = None
-    pid: Optional[int] = None
-
-    if isinstance(process, int):
-        pid = process
-    elif hasattr(process, "pid"):
-        # Duck-type any object that exposes a pid (e.g., multiprocessing.Process or Mock)
-        proc = process
-        try:
-            pid = int(getattr(proc, "pid"))
-        except Exception as e:
-            raise AttributeError(f"Invalid process-like object without usable pid: {e}")
-    else:
-        raise AttributeError(
-            "kill_pipeline_process_group expects a multiprocessing.Process or a PID int (process-like object with .pid)"
-        )
-
-    # If we have a Process handle and it's already dead, nothing to do
-    if proc is not None and hasattr(proc, "is_alive") and not proc.is_alive():
-        _safe_log(logging.DEBUG, "Process already terminated")
-        return
-
-    if pid is None:
-        # Defensive guard; should not happen
-        raise AttributeError("Unable to determine PID for process group termination")
-
-    _safe_log(logging.INFO, f"Terminating pipeline process group (PID: {pid})")
-    try:
-        # Send graceful termination to the entire process group
-        os.killpg(os.getpgid(pid), signal.SIGTERM)
-
-        # If we have a Process handle, give it a chance to exit cleanly
-        if proc is not None and hasattr(proc, "join"):
+    finally:
+        # Best-effort: if we created a broker here and the pipeline exits normally,
+        # attempt a graceful terminate. In failure/termination paths the process group kill
+        # from parent or signal handler will take care of it.
+        if broker_proc is not None:
             try:
-                proc.join(timeout=5.0)
+                if hasattr(broker_proc, "is_alive") and broker_proc.is_alive():
+                    broker_proc.terminate()
             except Exception:
                 pass
-            still_alive = getattr(proc, "is_alive", lambda: True)()
-        else:
-            # Without a handle, provide a small grace period
-            time.sleep(2.0)
-            # Best-effort check: if getpgid fails, it's gone
-            try:
-                _ = os.getpgid(pid)
-                still_alive = True
-            except Exception:
-                still_alive = False
 
-        if still_alive:
-            _safe_log(logging.WARNING, "Process group did not terminate gracefully, using SIGKILL")
-            try:
-                os.killpg(os.getpgid(pid), signal.SIGKILL)
-            finally:
-                if proc is not None and hasattr(proc, "join"):
-                    try:
-                        proc.join(timeout=3.0)
-                    except Exception:
-                        pass
-
-    except (ProcessLookupError, OSError) as e:
-        # Process or group may already be gone
-        _safe_log(logging.DEBUG, f"Process group already terminated or not found: {e}")
+
+def kill_pipeline_process_group(process: multiprocessing.Process) -> None:
+    """Backward-compatible shim that delegates to process.termination implementation."""
+    _safe_log(logging.DEBUG, "Delegating kill_pipeline_process_group to process.termination module")
+    _kill_pipeline_process_group(process)

nv_ingest/framework/orchestration/process/lifecycle.py

@@ -11,11 +11,16 @@ using the configured strategy pattern.
 """
 
 import logging
+import atexit
+import multiprocessing
+import os
+import signal
 from typing import Optional
 
 from nv_ingest.pipeline.pipeline_schema import PipelineConfigSchema
 from nv_ingest.framework.orchestration.execution.options import ExecutionOptions, ExecutionResult
 from nv_ingest.framework.orchestration.process.strategies import ProcessExecutionStrategy
+from nv_ingest.framework.orchestration.process.strategies import SubprocessStrategy
 from nv_ingest.framework.orchestration.process.dependent_services import start_simple_message_broker
 
 logger = logging.getLogger(__name__)
@@ -45,6 +50,8 @@ class PipelineLifecycleManager:
             The strategy to use for pipeline execution.
         """
         self.strategy = strategy
+        # Track broker process so we can terminate it during teardown
+        self._broker_process: Optional[multiprocessing.Process] = None
 
     def start(self, config: PipelineConfigSchema, options: ExecutionOptions) -> ExecutionResult:
         """
@@ -74,8 +81,18 @@ class PipelineLifecycleManager:
         """
         logger.info("Starting pipeline lifecycle")
 
+        # If running pipeline in a subprocess and broker is enabled, ensure the broker
+        # is launched in the child process group by signaling via environment variable
+        prev_env = None
+        set_env = False
+        if getattr(config, "pipeline", None) and getattr(config.pipeline, "launch_simple_broker", False):
+            if isinstance(self.strategy, SubprocessStrategy):
+                prev_env = os.environ.get("NV_INGEST_BROKER_IN_SUBPROCESS")
+                os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"] = "1"
+                set_env = True
+
         try:
-            # Start message broker if configured
+            # Start message broker if configured (may defer to subprocess based on env)
             self._setup_message_broker(config)
 
             # Execute pipeline using the configured strategy
@@ -87,6 +104,15 @@ class PipelineLifecycleManager:
         except Exception as e:
             logger.error(f"Failed to start pipeline lifecycle: {e}")
             raise RuntimeError(f"Pipeline startup failed: {e}") from e
+        finally:
+            if set_env:
+                if prev_env is None:
+                    try:
+                        del os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"]
+                    except KeyError:
+                        pass
+                else:
+                    os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"] = prev_env
 
     def _setup_message_broker(self, config: PipelineConfigSchema) -> None:
         """
@@ -98,8 +124,21 @@ class PipelineLifecycleManager:
             Pipeline configuration containing broker settings.
         """
         if config.pipeline.launch_simple_broker:
+            # If requested to launch broker inside the subprocess, skip here
+            if os.environ.get("NV_INGEST_BROKER_IN_SUBPROCESS") == "1":
+                logger.info("Deferring SimpleMessageBroker launch to subprocess")
+                return
             logger.info("Starting simple message broker")
-            start_simple_message_broker({})
+            # Start the broker and retain a handle for cleanup.
+            # Use defaults (host=0.0.0.0, port=7671) as set by the broker implementation.
+            try:
+                self._broker_process = start_simple_message_broker({})
+                # Ensure cleanup at interpreter shutdown in case caller forgets
+                atexit.register(self._terminate_broker_atexit)
+                logger.info(f"SimpleMessageBroker started (pid={getattr(self._broker_process, 'pid', None)})")
+            except Exception as e:
+                logger.error(f"Failed to start SimpleMessageBroker: {e}")
+                raise
         else:
             logger.debug("Simple broker launch not required")
 
@@ -109,6 +148,8 @@ class PipelineLifecycleManager:
 
         This method provides a hook for future pipeline stopping functionality.
         Currently, pipeline stopping is handled by the individual interfaces.
+        Additionally, it ensures any dependent services (like the simple
+        message broker) are terminated to avoid lingering processes.
 
         Parameters
         ----------
@@ -116,7 +157,58 @@ class PipelineLifecycleManager:
             Identifier of the pipeline to stop. Currently unused.
         """
         logger.info("Pipeline stop requested")
-        # TODO: Implement pipeline stopping logic when needed
-        # This would involve coordinating with the execution strategy
-        # to gracefully shut down running pipelines
-        pass
+        # Best-effort termination of broker if we started one
+        self._terminate_broker()
+
+    # --- Internal helpers ---
+    def _terminate_broker_atexit(self) -> None:
+        """Atexit-safe broker termination.
+
+        Avoids raising exceptions during interpreter shutdown.
+        """
+        try:
+            self._terminate_broker()
+        except Exception:
+            # Swallow errors at atexit to avoid noisy shutdowns
+            pass
+
+    def _terminate_broker(self) -> None:
+        """Terminate the SimpleMessageBroker process if running."""
+        proc = self._broker_process
+        if not proc:
+            return
+        try:
+            if hasattr(proc, "is_alive") and not proc.is_alive():
+                return
+        except Exception:
+            # If querying state fails, continue with termination attempt
+            pass
+
+        pid = getattr(proc, "pid", None)
+        logger.info(f"Stopping SimpleMessageBroker (pid={pid})")
+        try:
+            # First, try graceful terminate
+            proc.terminate()
+            try:
+                proc.join(timeout=3.0)
+            except Exception:
+                pass
+
+            # If still alive, escalate to SIGKILL on the single process
+            still_alive = False
+            try:
+                still_alive = hasattr(proc, "is_alive") and proc.is_alive()
+            except Exception:
+                still_alive = True
+            if still_alive and pid is not None:
+                try:
+                    os.kill(pid, signal.SIGKILL)
+                except Exception:
+                    pass
+                try:
+                    proc.join(timeout=2.0)
+                except Exception:
+                    pass
+        finally:
+            # Clear handle to avoid repeated attempts
+            self._broker_process = None

nv_ingest/framework/orchestration/process/strategies.py

@@ -11,6 +11,7 @@ Strategy pattern for clean separation of execution concerns.
 """
 
 import atexit
+import os
 import logging
 import multiprocessing
 import time
@@ -25,6 +26,8 @@ from nv_ingest.framework.orchestration.ray.primitives.ray_pipeline import (
 from nv_ingest.framework.orchestration.process.execution import (
     launch_pipeline,
     run_pipeline_process,
+)
+from nv_ingest.framework.orchestration.process.termination import (
     kill_pipeline_process_group,
 )
 
@@ -140,16 +143,45 @@ class SubprocessStrategy(ProcessExecutionStrategy):
             daemon=False,
         )
 
-        process.start()
+        # Hint to the lifecycle manager to skip starting the broker in the parent
+        prev_val = os.environ.get("NV_INGEST_BROKER_IN_SUBPROCESS")
+        os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"] = "1"
+        try:
+            process.start()
+        finally:
+            # Restore original env to avoid affecting other code paths
+            if prev_val is None:
+                try:
+                    del os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"]
+                except KeyError:
+                    pass
+            else:
+                os.environ["NV_INGEST_BROKER_IN_SUBPROCESS"] = prev_val
         interface = RayPipelineSubprocessInterface(process)
 
         if options.block:
-            # Block until subprocess completes
+            # Block until subprocess completes, handling Ctrl+C to ensure teardown
             start_time = time.time()
             logger.info("Waiting for subprocess pipeline to complete...")
-            process.join()
-            logger.info("Pipeline subprocess completed.")
+            try:
+                process.join()
+            except KeyboardInterrupt:
+                logger.info("KeyboardInterrupt in parent; terminating subprocess group...")
+                try:
+                    pid = int(process.pid)
+                    kill_pipeline_process_group(pid)
+                finally:
+                    # Best-effort wait for process to exit
+                    try:
+                        process.join(timeout=5.0)
+                    except Exception:
+                        pass
+            finally:
+                logger.info("Pipeline subprocess completed or terminated.")
             elapsed_time = time.time() - start_time
+            # If process ended with failure, surface it
+            if hasattr(process, "exitcode") and process.exitcode not in (0, None):
+                raise RuntimeError(f"Pipeline subprocess exited with code {process.exitcode}")
             return ExecutionResult(interface=None, elapsed_time=elapsed_time)
         else:
             # Return interface for non-blocking execution

nv_ingest/framework/orchestration/process/termination.py

@@ -0,0 +1,107 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Process termination utilities, isolated to avoid circular imports.
+
+This module provides functions to terminate a process and its entire process
+group safely, without depending on pipeline construction or Ray types.
+"""
+
+import logging
+import os
+import signal
+import time
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+def _safe_log(level: int, msg: str) -> None:
+    """Best-effort logging that won't crash during interpreter shutdown."""
+    try:
+        logger.log(level, msg)
+    except Exception:
+        try:
+            # Fallback to stderr if available
+            import sys
+
+            if hasattr(sys, "__stderr__") and sys.__stderr__:
+                sys.__stderr__.write(msg + "\n")
+                sys.__stderr__.flush()
+        except Exception:
+            pass
+
+
+def kill_pipeline_process_group(process) -> None:
+    """
+    Kill a process and its entire process group.
+
+    Accepts either a multiprocessing.Process-like object exposing a ``pid`` attribute
+    or a raw PID integer. Sends SIGTERM to the process group first, and escalates
+    to SIGKILL if it does not terminate within a short grace period.
+
+    Parameters
+    ----------
+    process : multiprocessing.Process | int
+        Process handle (or a raw PID int) for the process whose process group should be terminated.
+    """
+    proc: Optional[object] = None
+    pid: Optional[int] = None
+
+    if isinstance(process, int):
+        pid = process
+    elif hasattr(process, "pid"):
+        proc = process
+        try:
+            pid = int(getattr(proc, "pid"))
+        except Exception as e:
+            raise AttributeError(f"Invalid process-like object without usable pid: {e}")
+    else:
+        raise AttributeError(
+            "kill_pipeline_process_group expects a multiprocessing.Process or a PID int (process-like object with .pid)"
+        )
+
+    if proc is not None and hasattr(proc, "is_alive") and not proc.is_alive():
+        _safe_log(logging.DEBUG, "Process already terminated")
+        return
+
+    if pid is None:
+        raise AttributeError("Unable to determine PID for process group termination")
+
+    _safe_log(logging.INFO, f"Terminating pipeline process group (PID: {pid})")
+
+    try:
+        # Send graceful termination to the entire process group
+        os.killpg(os.getpgid(pid), signal.SIGTERM)
+
+        # If we have a Process handle, give it a chance to exit cleanly
+        if proc is not None and hasattr(proc, "join"):
+            try:
+                proc.join(timeout=5.0)
+            except Exception:
+                pass
+            still_alive = getattr(proc, "is_alive", lambda: True)()
+        else:
+            # Without a handle, provide a small grace period
+            time.sleep(2.0)
+            try:
+                _ = os.getpgid(pid)
+                still_alive = True
+            except Exception:
+                still_alive = False
+
+        if still_alive:
+            _safe_log(logging.WARNING, "Process group did not terminate gracefully, using SIGKILL")
+            try:
+                os.killpg(os.getpgid(pid), signal.SIGKILL)
+            finally:
+                if proc is not None and hasattr(proc, "join"):
+                    try:
+                        proc.join(timeout=3.0)
+                    except Exception:
+                        pass
+
+    except (ProcessLookupError, OSError) as e:
+        _safe_log(logging.DEBUG, f"Process group already terminated or not found: {e}")