nv-ingest 2025.8.4.dev20250804__py3-none-any.whl → 2025.12.10.dev20251210__py3-none-any.whl

nv_ingest/framework/orchestration/execution/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0

nv_ingest/framework/orchestration/execution/helpers.py

@@ -0,0 +1,85 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Helper functions for pipeline execution configuration.
+
+This module contains generic helper functions for converting individual parameters
+into structured configuration objects, supporting the declarative execution architecture.
+"""
+
+from typing import Optional, TextIO
+
+from nv_ingest.framework.orchestration.execution.options import PipelineRuntimeOverrides, ExecutionOptions
+from nv_ingest.framework.orchestration.process.strategies import ProcessExecutionStrategy, create_execution_strategy
+
+
+def create_runtime_overrides(
+    disable_dynamic_scaling: Optional[bool], dynamic_memory_threshold: Optional[float]
+) -> PipelineRuntimeOverrides:
+    """
+    Create runtime override object from individual parameters.
+
+    This function converts the individual override parameters into
+    a structured PipelineRuntimeOverrides object for declarative processing.
+
+    Parameters
+    ----------
+    disable_dynamic_scaling : Optional[bool]
+        Dynamic scaling override value.
+    dynamic_memory_threshold : Optional[float]
+        Memory threshold override value.
+
+    Returns
+    -------
+    PipelineRuntimeOverrides
+        Structured override object containing the provided values.
+    """
+    return PipelineRuntimeOverrides(
+        disable_dynamic_scaling=disable_dynamic_scaling, dynamic_memory_threshold=dynamic_memory_threshold
+    )
+
+
+def create_execution_options(block: bool, stdout: Optional[TextIO], stderr: Optional[TextIO]) -> ExecutionOptions:
+    """
+    Create execution options object from individual parameters.
+
+    This function converts individual execution parameters into
+    a structured ExecutionOptions object for declarative processing.
+
+    Parameters
+    ----------
+    block : bool
+        Whether to block until pipeline completion.
+    stdout : Optional[TextIO]
+        Output stream for subprocess redirection.
+    stderr : Optional[TextIO]
+        Error stream for subprocess redirection.
+
+    Returns
+    -------
+    ExecutionOptions
+        Structured options object containing the provided values.
+    """
+    return ExecutionOptions(block=block, stdout=stdout, stderr=stderr)
+
+
+def select_execution_strategy(run_in_subprocess: bool) -> ProcessExecutionStrategy:
+    """
+    Select appropriate execution strategy based on parameters.
+
+    This function encapsulates the logic for choosing between
+    in-process and subprocess execution strategies.
+
+    Parameters
+    ----------
+    run_in_subprocess : bool
+        Whether to run in a subprocess.
+
+    Returns
+    -------
+    ProcessExecutionStrategy
+        Configured execution strategy instance.
+    """
+    return create_execution_strategy(run_in_subprocess=run_in_subprocess)

nv_ingest/framework/orchestration/execution/options.py

@@ -0,0 +1,112 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Data classes for pipeline execution configuration and options.
+
+This module defines declarative data structures for configuring pipeline execution,
+replacing imperative parameter passing with structured configuration objects.
+"""
+
+from dataclasses import dataclass
+from typing import Optional, TextIO, Union
+
+from nv_ingest.framework.orchestration.ray.primitives.ray_pipeline import (
+    RayPipelineInterface,
+    RayPipelineSubprocessInterface,
+)
+
+
+@dataclass
+class PipelineRuntimeOverrides:
+    """
+    Runtime parameter overrides for pipeline configuration.
+
+    These overrides are applied to the base pipeline configuration
+    to customize runtime behavior without modifying the source config.
+
+    Attributes
+    ----------
+    disable_dynamic_scaling : Optional[bool]
+        Override for dynamic scaling behavior. If provided, overrides
+        the pipeline config's disable_dynamic_scaling setting.
+    dynamic_memory_threshold : Optional[float]
+        Override for memory threshold used in dynamic scaling decisions.
+        Must be between 0.0 and 1.0 if provided.
+    """
+
+    disable_dynamic_scaling: Optional[bool] = None
+    dynamic_memory_threshold: Optional[float] = None
+
+    def __post_init__(self):
+        """Validate override values."""
+        if self.dynamic_memory_threshold is not None:
+            if not (0.0 <= self.dynamic_memory_threshold <= 1.0):
+                raise ValueError(
+                    f"dynamic_memory_threshold must be between 0.0 and 1.0, " f"got {self.dynamic_memory_threshold}"
+                )
+
+
+@dataclass
+class ExecutionOptions:
+    """
+    Options controlling pipeline execution behavior.
+
+    These options determine how the pipeline is executed (blocking vs non-blocking)
+    and where output is directed for subprocess execution.
+
+    Attributes
+    ----------
+    block : bool
+        If True, blocks until pipeline completes. If False, returns
+        immediately with a control interface.
+    stdout : Optional[TextIO]
+        Stream for subprocess stdout redirection. Only used when
+        run_in_subprocess=True. If None, redirected to /dev/null.
+    stderr : Optional[TextIO]
+        Stream for subprocess stderr redirection. Only used when
+        run_in_subprocess=True. If None, redirected to /dev/null.
+    """
+
+    block: bool = True
+    stdout: Optional[TextIO] = None
+    stderr: Optional[TextIO] = None
+
+
+@dataclass
+class ExecutionResult:
+    """
+    Result of pipeline execution containing interface and timing information.
+
+    This class encapsulates the results of pipeline execution and provides
+    methods to convert to the legacy return format for backward compatibility.
+
+    Attributes
+    ----------
+    interface : Union[RayPipelineInterface, RayPipelineSubprocessInterface, None]
+        Pipeline control interface. None for blocking subprocess execution.
+    elapsed_time : Optional[float]
+        Total execution time in seconds. Only set for blocking execution.
+    """
+
+    interface: Union[RayPipelineInterface, RayPipelineSubprocessInterface, None]
+    elapsed_time: Optional[float] = None
+
+    def get_return_value(self) -> Union[RayPipelineInterface, float, RayPipelineSubprocessInterface]:
+        """
+        Convert to legacy return format for backward compatibility.
+
+        Returns
+        -------
+        Union[RayPipelineInterface, float, RayPipelineSubprocessInterface]
+            - If blocking execution: returns elapsed time (float)
+            - If non-blocking execution: returns pipeline interface
+        """
+        if self.elapsed_time is not None:
+            return self.elapsed_time
+        elif self.interface is not None:
+            return self.interface
+        else:
+            # This should not happen in normal execution
+            raise RuntimeError("ExecutionResult has neither interface nor elapsed_time")

nv_ingest/framework/orchestration/process/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0

nv_ingest/framework/orchestration/process/dependent_services.py

@@ -0,0 +1,84 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Dependent services management for pipeline orchestration.
+
+This module contains utilities for starting and managing dependent services
+that the pipeline requires, such as message brokers and other infrastructure.
+"""
+
+import logging
+import os
+import multiprocessing
+import socket
+from nv_ingest_api.util.message_brokers.simple_message_broker.broker import SimpleMessageBroker
+
+logger = logging.getLogger(__name__)
+
+
+def _broker_server_target(host, port, max_queue_size):
+    """
+    Target function to be run in a separate process for the SimpleMessageBroker.
+    """
+    server = SimpleMessageBroker(host, port, max_queue_size)
+    try:
+        server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    except Exception:
+        pass
+    server.serve_forever()
+
+
+def start_simple_message_broker(broker_client: dict) -> multiprocessing.Process:
+    """
+    Starts a SimpleMessageBroker server in a separate process.
+
+    Parameters
+    ----------
+    broker_client : dict
+        Broker configuration. Expected keys include:
+          - "port": the port to bind the server to,
+          - "broker_params": optionally including "max_queue_size",
+          - and any other parameters required by SimpleMessageBroker.
+
+    Returns
+    -------
+    multiprocessing.Process
+        The process running the SimpleMessageBroker server.
+    """
+
+    # Resolve host/port early for pre-flight checks
+    broker_params = broker_client.get("broker_params", {})
+    max_queue_size = broker_params.get("max_queue_size", 10000)
+    server_host = broker_client.get("host", "0.0.0.0")
+    server_port = broker_client.get("port", 7671)
+
+    # Pre-flight: if something is already listening on the target port, do not spawn another broker.
+    # This avoids noisy stack traces from a failing child process when tests/pipeline are run repeatedly.
+    def _is_port_open(host: str, port: int) -> bool:
+        check_host = "127.0.0.1" if host in ("0.0.0.0", "::") else host
+        try:
+            with socket.create_connection((check_host, port), timeout=0.5):
+                return True
+        except Exception:
+            return False
+
+    if _is_port_open(server_host, server_port):
+        logger.warning(
+            f"SimpleMessageBroker port already in use at {server_host}:{server_port}; "
+            f"continuing to spawn a broker process (tests expect a Process to be returned)"
+        )
+
+    p = multiprocessing.Process(
+        target=_broker_server_target,
+        args=(server_host, server_port, max_queue_size),
+        daemon=True,
+    )
+    # If we're launching from inside the pipeline subprocess, mark daemon so the
+    # broker dies automatically when the subprocess exits.
+    p.daemon = os.environ.get("NV_INGEST_BROKER_IN_SUBPROCESS") == "1"
+    p.start()
+    logger.info(f"Started SimpleMessageBroker server in separate process on port {server_port}")
+
+    return p