nv-ingest 2025.8.14.dev20250814__tar.gz → 2025.8.15.dev20250815__tar.gz

{nv_ingest-2025.8.14.dev20250814 → nv_ingest-2025.8.15.dev20250815}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.8.14.dev20250814
+Version: 2025.8.15.dev20250815
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

nv_ingest-2025.8.15.dev20250815/nv_ingest/framework/orchestration/execution/helpers.py

@@ -0,0 +1,85 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Helper functions for pipeline execution configuration.
+
+This module contains generic helper functions for converting individual parameters
+into structured configuration objects, supporting the declarative execution architecture.
+"""
+
+from typing import Optional, TextIO
+
+from nv_ingest.framework.orchestration.execution.options import PipelineRuntimeOverrides, ExecutionOptions
+from nv_ingest.framework.orchestration.process.strategies import ProcessExecutionStrategy, create_execution_strategy
+
+
+def create_runtime_overrides(
+    disable_dynamic_scaling: Optional[bool], dynamic_memory_threshold: Optional[float]
+) -> PipelineRuntimeOverrides:
+    """
+    Create runtime override object from individual parameters.
+
+    This function converts the individual override parameters into
+    a structured PipelineRuntimeOverrides object for declarative processing.
+
+    Parameters
+    ----------
+    disable_dynamic_scaling : Optional[bool]
+        Dynamic scaling override value.
+    dynamic_memory_threshold : Optional[float]
+        Memory threshold override value.
+
+    Returns
+    -------
+    PipelineRuntimeOverrides
+        Structured override object containing the provided values.
+    """
+    return PipelineRuntimeOverrides(
+        disable_dynamic_scaling=disable_dynamic_scaling, dynamic_memory_threshold=dynamic_memory_threshold
+    )
+
+
+def create_execution_options(block: bool, stdout: Optional[TextIO], stderr: Optional[TextIO]) -> ExecutionOptions:
+    """
+    Create execution options object from individual parameters.
+
+    This function converts individual execution parameters into
+    a structured ExecutionOptions object for declarative processing.
+
+    Parameters
+    ----------
+    block : bool
+        Whether to block until pipeline completion.
+    stdout : Optional[TextIO]
+        Output stream for subprocess redirection.
+    stderr : Optional[TextIO]
+        Error stream for subprocess redirection.
+
+    Returns
+    -------
+    ExecutionOptions
+        Structured options object containing the provided values.
+    """
+    return ExecutionOptions(block=block, stdout=stdout, stderr=stderr)
+
+
+def select_execution_strategy(run_in_subprocess: bool) -> ProcessExecutionStrategy:
+    """
+    Select appropriate execution strategy based on parameters.
+
+    This function encapsulates the logic for choosing between
+    in-process and subprocess execution strategies.
+
+    Parameters
+    ----------
+    run_in_subprocess : bool
+        Whether to run in a subprocess.
+
+    Returns
+    -------
+    ProcessExecutionStrategy
+        Configured execution strategy instance.
+    """
+    return create_execution_strategy(run_in_subprocess=run_in_subprocess)

nv_ingest-2025.8.15.dev20250815/nv_ingest/framework/orchestration/execution/options.py

@@ -0,0 +1,112 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Data classes for pipeline execution configuration and options.
+
+This module defines declarative data structures for configuring pipeline execution,
+replacing imperative parameter passing with structured configuration objects.
+"""
+
+from dataclasses import dataclass
+from typing import Optional, TextIO, Union
+
+from nv_ingest.framework.orchestration.ray.primitives.ray_pipeline import (
+    RayPipelineInterface,
+    RayPipelineSubprocessInterface,
+)
+
+
+@dataclass
+class PipelineRuntimeOverrides:
+    """
+    Runtime parameter overrides for pipeline configuration.
+
+    These overrides are applied to the base pipeline configuration
+    to customize runtime behavior without modifying the source config.
+
+    Attributes
+    ----------
+    disable_dynamic_scaling : Optional[bool]
+        Override for dynamic scaling behavior. If provided, overrides
+        the pipeline config's disable_dynamic_scaling setting.
+    dynamic_memory_threshold : Optional[float]
+        Override for memory threshold used in dynamic scaling decisions.
+        Must be between 0.0 and 1.0 if provided.
+    """
+
+    disable_dynamic_scaling: Optional[bool] = None
+    dynamic_memory_threshold: Optional[float] = None
+
+    def __post_init__(self):
+        """Validate override values."""
+        if self.dynamic_memory_threshold is not None:
+            if not (0.0 <= self.dynamic_memory_threshold <= 1.0):
+                raise ValueError(
+                    f"dynamic_memory_threshold must be between 0.0 and 1.0, " f"got {self.dynamic_memory_threshold}"
+                )
+
+
+@dataclass
+class ExecutionOptions:
+    """
+    Options controlling pipeline execution behavior.
+
+    These options determine how the pipeline is executed (blocking vs non-blocking)
+    and where output is directed for subprocess execution.
+
+    Attributes
+    ----------
+    block : bool
+        If True, blocks until pipeline completes. If False, returns
+        immediately with a control interface.
+    stdout : Optional[TextIO]
+        Stream for subprocess stdout redirection. Only used when
+        run_in_subprocess=True. If None, redirected to /dev/null.
+    stderr : Optional[TextIO]
+        Stream for subprocess stderr redirection. Only used when
+        run_in_subprocess=True. If None, redirected to /dev/null.
+    """
+
+    block: bool = True
+    stdout: Optional[TextIO] = None
+    stderr: Optional[TextIO] = None
+
+
+@dataclass
+class ExecutionResult:
+    """
+    Result of pipeline execution containing interface and timing information.
+
+    This class encapsulates the results of pipeline execution and provides
+    methods to convert to the legacy return format for backward compatibility.
+
+    Attributes
+    ----------
+    interface : Union[RayPipelineInterface, RayPipelineSubprocessInterface, None]
+        Pipeline control interface. None for blocking subprocess execution.
+    elapsed_time : Optional[float]
+        Total execution time in seconds. Only set for blocking execution.
+    """
+
+    interface: Union[RayPipelineInterface, RayPipelineSubprocessInterface, None]
+    elapsed_time: Optional[float] = None
+
+    def get_return_value(self) -> Union[RayPipelineInterface, float, RayPipelineSubprocessInterface]:
+        """
+        Convert to legacy return format for backward compatibility.
+
+        Returns
+        -------
+        Union[RayPipelineInterface, float, RayPipelineSubprocessInterface]
+            - If blocking execution: returns elapsed time (float)
+            - If non-blocking execution: returns pipeline interface
+        """
+        if self.elapsed_time is not None:
+            return self.elapsed_time
+        elif self.interface is not None:
+            return self.interface
+        else:
+            # This should not happen in normal execution
+            raise RuntimeError("ExecutionResult has neither interface nor elapsed_time")

nv_ingest-2025.8.15.dev20250815/nv_ingest/framework/orchestration/process/dependent_services.py

@@ -0,0 +1,55 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Dependent services management for pipeline orchestration.
+
+This module contains utilities for starting and managing dependent services
+that the pipeline requires, such as message brokers and other infrastructure.
+"""
+
+import logging
+import multiprocessing
+import socket
+from nv_ingest_api.util.message_brokers.simple_message_broker.broker import SimpleMessageBroker
+
+logger = logging.getLogger(__name__)
+
+
+def start_simple_message_broker(broker_client: dict) -> multiprocessing.Process:
+    """
+    Starts a SimpleMessageBroker server in a separate process.
+
+    Parameters
+    ----------
+    broker_client : dict
+        Broker configuration. Expected keys include:
+          - "port": the port to bind the server to,
+          - "broker_params": optionally including "max_queue_size",
+          - and any other parameters required by SimpleMessageBroker.
+
+    Returns
+    -------
+    multiprocessing.Process
+        The process running the SimpleMessageBroker server.
+    """
+
+    def broker_server():
+        # Use max_queue_size from broker_params or default to 10000.
+        broker_params = broker_client.get("broker_params", {})
+        max_queue_size = broker_params.get("max_queue_size", 10000)
+        server_host = broker_client.get("host", "0.0.0.0")
+        server_port = broker_client.get("port", 7671)
+        # Optionally, set socket options here for reuse.
+        server = SimpleMessageBroker(server_host, server_port, max_queue_size)
+        # Enable address reuse on the server socket.
+        server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        server.serve_forever()
+
+    p = multiprocessing.Process(target=broker_server)
+    p.daemon = False
+    p.start()
+    logger.info(f"Started SimpleMessageBroker server in separate process on port {broker_client.get('port', 7671)}")
+
+    return p