nv-ingest 2025.5.21.dev20250521__py3-none-any.whl → 2025.5.29.dev20250529__py3-none-any.whl

nv_ingest/framework/orchestration/ray/stages/extractors/html_extractor.py

@@ -0,0 +1,82 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+
+import logging
+
+import ray
+
+from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
+from nv_ingest.framework.util.flow_control import filter_by_task
+from nv_ingest_api.internal.extract.html.html_extractor import extract_markdown_from_html_internal
+from nv_ingest_api.internal.primitives.ingest_control_message import remove_task_by_type, IngestControlMessage
+from nv_ingest_api.internal.primitives.tracing.tagging import traceable
+from nv_ingest_api.internal.schemas.extract.extract_html_schema import HtmlExtractorSchema
+from nv_ingest_api.util.exception_handlers.decorators import (
+    nv_ingest_node_failure_try_except,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@ray.remote
+class HtmlExtractorStage(RayActorStage):
+    """
+    A Ray actor stage that extracts text in markdown format from html content.
+
+    It expects an IngestControlMessage containing a DataFrame with html content. It then:
+      1. Removes the "html_content_extract" task from the message.
+      2. Calls the html extraction logic (via extract_markdown_from_html_internal) using a validated configuration.
+      3. Updates the message payload with the extracted text DataFrame.
+    """
+
+    def __init__(self, config: HtmlExtractorSchema) -> None:
+        super().__init__(config, log_to_stdout=False)
+        try:
+            self.validated_config = config
+            self._logger.info("HtmlExtractorStage configuration validated successfully.")
+        except Exception as e:
+            self._logger.exception(f"Error validating Html Extractor config: {e}")
+            raise
+
+    @traceable("html_extractor")
+    @filter_by_task(required_tasks=[("extract", {"document_type": "html"})])
+    @nv_ingest_node_failure_try_except(annotation_id="html_extractor", raise_on_failure=False)
+    def on_data(self, control_message: IngestControlMessage) -> IngestControlMessage:
+        """
+        Process the control message by extracting content from html.
+
+        Parameters
+        ----------
+        control_message : IngestControlMessage
+            The message containing a DataFrame payload with html content.
+
+        Returns
+        -------
+        IngestControlMessage
+            The updated message with extracted content.
+        """
+        self._logger.debug("HtmlExtractorStage.on_data: Starting html extraction process.")
+
+        # Extract the DataFrame payload.
+        df_ledger = control_message.payload()
+        self._logger.debug("Extracted payload with %d rows.", len(df_ledger))
+
+        # Remove the "html_content_extract" task from the message to obtain task-specific configuration.
+        task_config = remove_task_by_type(control_message, "extract")
+        self._logger.debug("Extracted task config: %s", task_config)
+
+        # Perform html content extraction.
+        new_df, extraction_info = extract_markdown_from_html_internal(
+            df_extraction_ledger=df_ledger,
+            task_config=task_config,
+            extraction_config=self.validated_config,
+            execution_trace_log=None,
+        )
+
+        # Update the message payload with the extracted text DataFrame.
+        control_message.payload(new_df)
+        control_message.set_metadata("html_extraction_info", extraction_info)
+
+        return control_message

nv_ingest/framework/orchestration/ray/stages/sources/message_broker_task_source.py

@@ -495,7 +495,7 @@ def start_simple_message_broker(broker_client: dict) -> multiprocessing.Process:
         server.serve_forever()
 
     p = multiprocessing.Process(target=broker_server)
-    p.daemon = True
+    p.daemon = False
     p.start()
     logger.info(f"Started SimpleMessageBroker server in separate process on port {broker_client['port']}")
 

nv_ingest/framework/orchestration/ray/util/pipeline/pid_controller.py

@@ -490,7 +490,7 @@ class ResourceConstraintManager:
         final_proposals_this_step = {}
 
         if not room_to_scale_up_to_global_caps:
-            logger.info(
+            logger.debug(
                 "[ConstraintMgr-Proportional] Global scaling beyond effective minimums is RESTRICTED "
                 "as SumOfEffectiveMins likely meets/exceeds a global Core/MaxReplica cap. "
                 "Proposed increases from initial current values will be nullified."
@@ -502,7 +502,7 @@ class ResourceConstraintManager:
                 if val_from_prior_phases > original_current_replicas:
                     final_proposals_this_step[name] = original_current_replicas
                     if val_from_prior_phases != original_current_replicas:
-                        logger.info(
+                        logger.debug(
                             f"[ConstraintMgr-{name}] Proportional: Scaling restricted. "
                             f"Nullified proposed increase from {original_current_replicas} to {val_from_prior_phases}. "
                             f"Setting to {original_current_replicas}."
@@ -618,7 +618,7 @@ class ResourceConstraintManager:
 
         # Apply reduction to the deltas
         if reduction_factor <= 0.001:  # Epsilon for float
-            logger.info(
+            logger.debug(
                 f"[ConstraintMgr-Proportional] Scale-up beyond effective minimums fully constrained by global limits. "
                 f"Reasons: {'; '.join(limiting_reasons) if limiting_reasons else 'None'}. "
                 f"Final ReductionFactor={reduction_factor:.3f}."
@@ -637,7 +637,7 @@ class ResourceConstraintManager:
                 )
 
         elif reduction_factor < 1.0:
-            logger.info(
+            logger.debug(
                 f"[ConstraintMgr-Proportional] Reducing requested scale-up (beyond effective_mins) by "
                 f"factor {reduction_factor:.3f}. "
                 f"Limiting Factors: {'; '.join(limiting_reasons)}."
@@ -654,7 +654,7 @@ class ResourceConstraintManager:
                         f"-> FinalVal={final_value_for_stage}"
                     )
         else:  # reduction_factor is ~1.0, meaning full requested increase (above effective_mins) is allowed
-            logger.info(
+            logger.debug(
                 "[ConstraintMgr-Proportional] Full requested scale-up (beyond effective_mins) "
                 "is permissible by global limits."
             )
@@ -713,7 +713,7 @@ class ResourceConstraintManager:
                     target = max(1, min_r)
                     final_target = min(target, max_r)
                     if final_target > 0:
-                        logger.info(
+                        logger.debug(
                             f"[ConstraintMgr-{name}] Forcing minimum {final_target} replica due to global wake-up."
                         )
                         final_adjustments[name] = final_target
@@ -740,19 +740,19 @@ class ResourceConstraintManager:
         num_queue_actors = num_edges
         total_ray_components_for_info = final_stage_replicas_total + num_queue_actors
 
-        logger.info("[ConstraintMgr] --- Final Decision & Constraint Summary ---")
+        logger.debug("[ConstraintMgr] --- Final Decision & Constraint Summary ---")
 
         # --- I. Overall Pipeline State ---
-        logger.info(f"[ConstraintMgr]   Pipeline Activity: {global_in_flight} tasks in-flight.")
-        logger.info(f"[ConstraintMgr]   Effective Min Replicas (Sum): {sum_of_effective_mins}")
-        logger.info(
+        logger.debug(f"[ConstraintMgr]   Pipeline Activity: {global_in_flight} tasks in-flight.")
+        logger.debug(f"[ConstraintMgr]   Effective Min Replicas (Sum): {sum_of_effective_mins}")
+        logger.debug(
             f"[ConstraintMgr]     └─ Global Scaling Beyond Mins Permitted? {can_globally_scale_beyond_effective_mins}"
         )
 
         # --- II. Final Component Counts ---
-        logger.info(f"[ConstraintMgr]   Final Stage Replicas: {final_stage_replicas_total} (Target for caps)")
-        logger.info(f"[ConstraintMgr]   Queue/Edge Actors   : {num_queue_actors} (Informational)")
-        logger.info(f"[ConstraintMgr]   Total Ray Components: {total_ray_components_for_info} (Informational)")
+        logger.debug(f"[ConstraintMgr]   Final Stage Replicas: {final_stage_replicas_total} (Target for caps)")
+        logger.debug(f"[ConstraintMgr]   Queue/Edge Actors   : {num_queue_actors} (Informational)")
+        logger.debug(f"[ConstraintMgr]   Total Ray Components: {total_ray_components_for_info} (Informational)")
 
         # --- III. Resource Limits & Projected Usage (for Stages) ---
         # Configured Limits
@@ -762,18 +762,18 @@ class ResourceConstraintManager:
         )
         eff_mem_limit_str = f"{self.effective_memory_limit_mb:.1f}MB"
 
-        logger.info("[ConstraintMgr]   Global Limits (Stages):")
-        logger.info(f"[ConstraintMgr]     ├─ MaxTotalReplicas  : {max_r_cfg_str}")
-        logger.info(
+        logger.debug("[ConstraintMgr]   Global Limits (Stages):")
+        logger.debug(f"[ConstraintMgr]     ├─ MaxTotalReplicas  : {max_r_cfg_str}")
+        logger.debug(
             f"[ConstraintMgr]     ├─ CoreBasedRepLimit : {core_based_limit_str} "
             f"(System EffCores: {self.available_cores if self.available_cores is not None else 'N/A'})"
         )
-        logger.info(f"[ConstraintMgr]     └─ EffectiveMemLimit : {eff_mem_limit_str} ")
+        logger.debug(f"[ConstraintMgr]     └─ EffectiveMemLimit : {eff_mem_limit_str} ")
 
         # Usage vs Limits
-        logger.info("[ConstraintMgr]   Projected Usage (Stages):")
-        logger.info(f"[ConstraintMgr]     ├─ Replicas          : {final_stage_replicas_total}")
-        logger.info(
+        logger.debug("[ConstraintMgr]   Projected Usage (Stages):")
+        logger.debug(f"[ConstraintMgr]     ├─ Replicas          : {final_stage_replicas_total}")
+        logger.debug(
             f"[ConstraintMgr]     └─ Memory            : {projected_final_memory_mb:.1f}MB "
             f"(Current: {current_global_memory_usage_mb:.1f}MB)"
         )
@@ -815,20 +815,20 @@ class ResourceConstraintManager:
             )
             unexpected_breaches_details.append(f"MemoryLimit: {status_mem}")
 
-        logger.info("[ConstraintMgr]   Limit Adherence (Stages):")
-        logger.info(f"[ConstraintMgr]     ├─ MaxTotalReplicas  : {status_max_r}")
-        logger.info(f"[ConstraintMgr]     ├─ CoreBasedRepLimit : {status_core_r}")
-        logger.info(f"[ConstraintMgr]     └─ EffectiveMemLimit : {status_mem}")
+        logger.debug("[ConstraintMgr]   Limit Adherence (Stages):")
+        logger.debug(f"[ConstraintMgr]     ├─ MaxTotalReplicas  : {status_max_r}")
+        logger.debug(f"[ConstraintMgr]     ├─ CoreBasedRepLimit : {status_core_r}")
+        logger.debug(f"[ConstraintMgr]     └─ EffectiveMemLimit : {status_mem}")
 
         if unexpected_breaches_details:
-            logger.warning(f"[ConstraintMgr]   └─ UNEXPECTED BREACHES: {'; '.join(unexpected_breaches_details)}")
+            logger.debug(f"[ConstraintMgr]   └─ UNEXPECTED BREACHES: {'; '.join(unexpected_breaches_details)}")
         else:
-            logger.info("[ConstraintMgr]   └─ All hard caps (beyond tolerated minimums/wake-up) appear respected.")
+            logger.debug("[ConstraintMgr]   └─ All hard caps (beyond tolerated minimums/wake-up) appear respected.")
 
         # --- V. Final Decisions Per Stage ---
-        logger.info("[ConstraintMgr]   Final Decisions (Per Stage):")
+        logger.debug("[ConstraintMgr]   Final Decisions (Per Stage):")
         if not final_adjustments:
-            logger.info("[ConstraintMgr]     └─ No stages to adjust.")
+            logger.debug("[ConstraintMgr]     └─ No stages to adjust.")
         else:
             # Determine max stage name length for alignment
             max_name_len = 0
@@ -843,12 +843,12 @@ class ResourceConstraintManager:
                 eff_min_str = f"(EffMin: {min_replicas if orig_prop else 'N/A'})"
 
                 # Basic alignment, can be improved with more sophisticated padding
-                logger.info(
+                logger.debug(
                     f"[ConstraintMgr]     └─ {stage_name:<{max_name_len}} : "
                     f"{count:<3} {pid_proposed_str} {current_str} {eff_min_str}"
                 )
 
-        logger.info("[ConstraintMgr] --- Constraint Summary END ---")
+        logger.debug("[ConstraintMgr] --- Constraint Summary END ---")
 
     # --- Public Method ---
 
@@ -863,7 +863,7 @@ class ResourceConstraintManager:
         Applies all configured constraints to initial replica proposals.
         (Docstring from previous version is fine)
         """
-        logger.info(
+        logger.debug(
             f"[ConstraintMgr] --- Applying Constraints START --- "
             f"GlobalInFlight={global_in_flight}, "
             f"CurrentGlobalMemMB={current_global_memory_usage_mb}, "
@@ -904,7 +904,7 @@ class ResourceConstraintManager:
             current_effective_mins[name] = eff_min
             sum_of_effective_mins += eff_min
 
-        logger.info(
+        logger.debug(
             f"[ConstraintMgr] Calculated Effective Minimums: TotalSum={sum_of_effective_mins}. "
             # f"IndividualMins: {current_effective_mins}" # Can be verbose
         )
@@ -985,5 +985,5 @@ class ResourceConstraintManager:
             can_globally_scale_up_stages,  # Pass this for context in logging
         )
 
-        logger.info("[ConstraintMgr] --- Applying Constraints END ---")
+        logger.debug("[ConstraintMgr] --- Applying Constraints END ---")
         return final_adjustments

nv_ingest/framework/orchestration/ray/util/pipeline/pipeline_builders.py

@@ -19,6 +19,7 @@ from nv_ingest.framework.orchestration.ray.util.pipeline.stage_builders import (
     add_image_extractor_stage,
     add_docx_extractor_stage,
     add_audio_extractor_stage,
+    add_html_extractor_stage,
     add_image_dedup_stage,
     add_image_filter_stage,
     add_table_extractor_stage,
@@ -53,7 +54,7 @@ def setup_ingestion_pipeline(pipeline: RayPipeline, ingest_config: Dict[str, Any
         export_config_to_env(ingest_config)
 
     current_level = logging.getLogger().getEffectiveLevel()
-    ray.init(
+    ray_context = ray.init(
         namespace="nv_ingest_ray",
         logging_level=current_level,
         ignore_reinit_error=True,
@@ -103,6 +104,7 @@ def setup_ingestion_pipeline(pipeline: RayPipeline, ingest_config: Dict[str, Any
     docx_extractor_stage_id = add_docx_extractor_stage(pipeline, default_cpu_count)
     pptx_extractor_stage_id = add_pptx_extractor_stage(pipeline, default_cpu_count)
     audio_extractor_stage_id = add_audio_extractor_stage(pipeline, default_cpu_count)
+    html_extractor_stage_id = add_html_extractor_stage(pipeline, default_cpu_count)
     ########################################################################################################
 
     ########################################################################################################
@@ -159,7 +161,8 @@ def setup_ingestion_pipeline(pipeline: RayPipeline, ingest_config: Dict[str, Any
     pipeline.make_edge(audio_extractor_stage_id, docx_extractor_stage_id, queue_size=ingest_edge_buffer_size)
     pipeline.make_edge(docx_extractor_stage_id, pptx_extractor_stage_id, queue_size=ingest_edge_buffer_size)
     pipeline.make_edge(pptx_extractor_stage_id, image_extractor_stage_id, queue_size=ingest_edge_buffer_size)
-    pipeline.make_edge(image_extractor_stage_id, infographic_extraction_stage_id, queue_size=ingest_edge_buffer_size)
+    pipeline.make_edge(image_extractor_stage_id, html_extractor_stage_id, queue_size=ingest_edge_buffer_size)
+    pipeline.make_edge(html_extractor_stage_id, infographic_extraction_stage_id, queue_size=ingest_edge_buffer_size)
 
     ###### Primitive Extractors ########
     pipeline.make_edge(infographic_extraction_stage_id, table_extraction_stage_id, queue_size=ingest_edge_buffer_size)
@@ -193,3 +196,5 @@ def setup_ingestion_pipeline(pipeline: RayPipeline, ingest_config: Dict[str, Any
     #    pipe.add_edge(sink_stage, otel_tracer_stage)
 
     # pipe.add_edge(otel_tracer_stage, completed_job_counter_stage)
+
+    return ray_context

nv_ingest/framework/orchestration/ray/util/pipeline/pipeline_runners.py

@@ -2,16 +2,26 @@
 # All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+import atexit
 import logging
+import multiprocessing
 import os
+import signal
+import sys
 import time
+from ctypes import CDLL, c_int
 from datetime import datetime
-from typing import Union, Tuple
+from typing import Union, Tuple, Optional, TextIO
 
 import ray
 from pydantic import BaseModel, ConfigDict
 
-from nv_ingest.framework.orchestration.ray.primitives.ray_pipeline import RayPipeline, ScalingConfig
+from nv_ingest.framework.orchestration.ray.primitives.ray_pipeline import (
+    RayPipeline,
+    ScalingConfig,
+    RayPipelineSubprocessInterface,
+    RayPipelineInterface,
+)
 from nv_ingest.framework.orchestration.ray.util.pipeline.pipeline_builders import setup_ingestion_pipeline
 
 logger = logging.getLogger(__name__)
@@ -33,6 +43,8 @@ class PipelineCreationSchema(BaseModel):
     including endpoints, API keys, and processing options.
     """
 
+    arrow_default_memory_pool: str = os.getenv("ARROW_DEFAULT_MEMORY_POOL", "system")
+
     # Audio processing settings
     audio_grpc_endpoint: str = os.getenv("AUDIO_GRPC_ENDPOINT", "grpc.nvcf.nvidia.com:443")
     audio_function_id: str = os.getenv("AUDIO_FUNCTION_ID", "1598d209-5e27-4d3c-8079-4751568b1081")
@@ -100,6 +112,112 @@ class PipelineCreationSchema(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
 
+def redirect_os_fds(stdout: Optional[TextIO] = None, stderr: Optional[TextIO] = None):
+    """
+    Redirect OS-level stdout (fd=1) and stderr (fd=2) to the given file-like objects,
+    or to /dev/null if not provided.
+
+    Parameters
+    ----------
+    stdout : Optional[TextIO]
+        Stream to receive OS-level stdout. If None, redirected to /dev/null.
+    stderr : Optional[TextIO]
+        Stream to receive OS-level stderr. If None, redirected to /dev/null.
+    """
+    devnull_fd = os.open(os.devnull, os.O_WRONLY)
+
+    if stdout is not None:
+        os.dup2(stdout.fileno(), 1)
+    else:
+        os.dup2(devnull_fd, 1)
+
+    if stderr is not None:
+        os.dup2(stderr.fileno(), 2)
+    else:
+        os.dup2(devnull_fd, 2)
+
+
+def set_pdeathsig(sig=signal.SIGKILL):
+    libc = CDLL("libc.so.6")
+    PR_SET_PDEATHSIG = 1
+    libc.prctl(PR_SET_PDEATHSIG, c_int(sig))
+
+
+def kill_pipeline_process_group(pid: int):
+    """
+    Kill the process group associated with the given PID, if it exists and is alive.
+
+    Parameters
+    ----------
+    pid : int
+        The PID of the process whose group should be killed.
+    """
+    try:
+        # Get the process group ID
+        pgid = os.getpgid(pid)
+
+        # Check if the group is still alive by sending signal 0
+        os.killpg(pgid, 0)  # Does not kill, just checks if it's alive
+
+        # If no exception, the group is alive — kill it
+        os.killpg(pgid, signal.SIGKILL)
+        print(f"Killed subprocess group {pgid}")
+
+    except ProcessLookupError:
+        print(f"Process group for PID {pid} no longer exists.")
+    except PermissionError:
+        print(f"Permission denied to kill process group for PID {pid}.")
+    except Exception as e:
+        print(f"Failed to kill subprocess group: {e}")
+
+
+def _run_pipeline_process(
+    ingest_config: PipelineCreationSchema,
+    disable_dynamic_scaling: Optional[bool],
+    dynamic_memory_threshold: Optional[float],
+    raw_stdout: Optional[TextIO] = None,
+    raw_stderr: Optional[TextIO] = None,
+):
+    """
+    Subprocess entrypoint to launch the pipeline. Redirects all output to the provided
+    file-like streams or /dev/null if not specified.
+
+    Parameters
+    ----------
+    ingest_config : PipelineCreationSchema
+        Validated pipeline configuration.
+    disable_dynamic_scaling : Optional[bool]
+        Whether to disable dynamic scaling.
+    dynamic_memory_threshold : Optional[float]
+        Threshold for triggering scaling.
+    raw_stdout : Optional[TextIO]
+        Destination for stdout. Defaults to /dev/null.
+    raw_stderr : Optional[TextIO]
+        Destination for stderr. Defaults to /dev/null.
+    """
+    # Set the death signal for the subprocess
+    set_pdeathsig()
+    os.setsid()  # Creates new process group so it can be SIGKILLed as a group
+
+    # Redirect OS-level file descriptors
+    redirect_os_fds(stdout=raw_stdout, stderr=raw_stderr)
+
+    # Redirect Python-level sys.stdout/sys.stderr
+    sys.stdout = raw_stdout or open(os.devnull, "w")
+    sys.stderr = raw_stderr or open(os.devnull, "w")
+
+    try:
+        _launch_pipeline(
+            ingest_config,
+            block=True,
+            disable_dynamic_scaling=disable_dynamic_scaling,
+            dynamic_memory_threshold=dynamic_memory_threshold,
+        )
+    except Exception as e:
+        sys.__stderr__.write(f"Subprocess pipeline run failed: {e}\n")
+        raise
+
+
 def _launch_pipeline(
     ingest_config: PipelineCreationSchema,
     block: bool,
@@ -122,7 +240,7 @@ def _launch_pipeline(
     start_abs = datetime.now()
 
     # Set up the ingestion pipeline
-    setup_ingestion_pipeline(pipeline, ingest_config.model_dump())
+    _ = setup_ingestion_pipeline(pipeline, ingest_config.model_dump())
 
     # Record setup time
     end_setup = start_run = datetime.now()
@@ -159,12 +277,100 @@ def _launch_pipeline(
 def run_pipeline(
     ingest_config: PipelineCreationSchema,
     block: bool = True,
-    disable_dynamic_scaling: bool = None,
-    dynamic_memory_threshold: float = None,
-) -> Union[RayPipeline, float]:
-    pipeline, total_elapsed = _launch_pipeline(ingest_config, block, disable_dynamic_scaling, dynamic_memory_threshold)
+    disable_dynamic_scaling: Optional[bool] = None,
+    dynamic_memory_threshold: Optional[float] = None,
+    run_in_subprocess: bool = False,
+    stdout: Optional[TextIO] = None,
+    stderr: Optional[TextIO] = None,
+) -> Union[RayPipelineInterface, float, RayPipelineSubprocessInterface]:
+    """
+    Launch and manage a pipeline, optionally in a subprocess.
+
+    This function is the primary entry point for executing a Ray pipeline,
+    either within the current process or in a separate Python subprocess.
+    It supports synchronous blocking execution or non-blocking lifecycle management,
+    and allows redirection of output to specified file-like objects.
+
+    Parameters
+    ----------
+    ingest_config : PipelineCreationSchema
+        The validated configuration object used to construct and launch the pipeline.
+    block : bool, default=True
+        If True, blocks until the pipeline completes.
+        If False, returns an interface to control the pipeline externally.
+    disable_dynamic_scaling : Optional[bool], default=None
+        If True, disables dynamic memory scaling. Overrides global configuration if set.
+        If None, uses the default or globally defined behavior.
+    dynamic_memory_threshold : Optional[float], default=None
+        The memory usage threshold (as a float between 0 and 1) that triggers autoscaling,
+        if dynamic scaling is enabled. Defaults to the globally configured value if None.
+    run_in_subprocess : bool, default=False
+        If True, launches the pipeline in a separate Python subprocess using `multiprocessing.Process`.
+        If False, runs the pipeline in the current process.
+    stdout : Optional[TextIO], default=None
+        Optional file-like stream to which subprocess stdout should be redirected.
+        If None, stdout is redirected to /dev/null.
+    stderr : Optional[TextIO], default=None
+        Optional file-like stream to which subprocess stderr should be redirected.
+        If None, stderr is redirected to /dev/null.
+
+    Returns
+    -------
+    Union[RayPipelineInterface, float, RayPipelineSubprocessInterface]
+        - If run in-process with `block=True`: returns elapsed time in seconds (float).
+        - If run in-process with `block=False`: returns a `RayPipelineInterface`.
+        - If run in subprocess with `block=False`: returns a `RayPipelineSubprocessInterface`.
+        - If run in subprocess with `block=True`: returns 0.0.
+
+    Raises
+    ------
+    RuntimeError
+        If the subprocess fails to start or exits with an error.
+    Exception
+        Any other exceptions raised during pipeline launch or configuration.
+    """
+    if run_in_subprocess:
+        logger.info("Launching pipeline in Python subprocess using multiprocessing.")
+
+        ctx = multiprocessing.get_context("fork")
+        process = ctx.Process(
+            target=_run_pipeline_process,
+            args=(
+                ingest_config,
+                disable_dynamic_scaling,
+                dynamic_memory_threshold,
+                stdout,  # raw_stdout
+                stderr,  # raw_stderr
+            ),
+            daemon=False,
+        )
+
+        process.start()
+
+        interface = RayPipelineSubprocessInterface(process)
+
+        if block:
+            start_time = time.time()
+            logger.info("Waiting for subprocess pipeline to complete...")
+            process.join()
+            logger.info("Pipeline subprocess completed.")
+            return time.time() - start_time
+        else:
+            logger.info(f"Pipeline subprocess started (PID={process.pid})")
+            atexit.register(lambda: kill_pipeline_process_group(process.pid))
+
+            return interface
+
+    # Run inline
+    pipeline, total_elapsed = _launch_pipeline(
+        ingest_config,
+        block=block,
+        disable_dynamic_scaling=disable_dynamic_scaling,
+        dynamic_memory_threshold=dynamic_memory_threshold,
+    )
 
     if block:
         logger.debug(f"Pipeline execution completed successfully in {total_elapsed:.2f} seconds.")
-
-    return pipeline
+        return total_elapsed
+    else:
+        return RayPipelineInterface(pipeline)

nv_ingest/framework/orchestration/ray/util/pipeline/stage_builders.py

@@ -23,6 +23,7 @@ from nv_ingest.framework.orchestration.ray.stages.extractors.infographic_extract
 from nv_ingest.framework.orchestration.ray.stages.extractors.pdf_extractor import PDFExtractorStage
 from nv_ingest.framework.orchestration.ray.stages.extractors.pptx_extractor import PPTXExtractorStage
 from nv_ingest.framework.orchestration.ray.stages.extractors.table_extractor import TableExtractorStage
+from nv_ingest.framework.orchestration.ray.stages.extractors.html_extractor import HtmlExtractorStage
 
 from nv_ingest.framework.orchestration.ray.stages.injectors.metadata_injector import MetadataInjectionStage
 from nv_ingest.framework.orchestration.ray.stages.mutate.image_dedup import ImageDedupStage
@@ -49,6 +50,7 @@ from nv_ingest_api.internal.schemas.extract.extract_image_schema import ImageCon
 from nv_ingest_api.internal.schemas.extract.extract_pdf_schema import PDFExtractorSchema
 from nv_ingest_api.internal.schemas.extract.extract_pptx_schema import PPTXExtractorSchema
 from nv_ingest_api.internal.schemas.extract.extract_table_schema import TableExtractorSchema
+from nv_ingest_api.internal.schemas.extract.extract_html_schema import HtmlExtractorSchema
 from nv_ingest_api.internal.schemas.mutate.mutate_image_dedup_schema import ImageDedupSchema
 from nv_ingest_api.internal.schemas.store.store_embedding_schema import EmbeddingStorageSchema
 from nv_ingest_api.internal.schemas.store.store_image_schema import ImageStorageModuleSchema
@@ -383,6 +385,19 @@ def add_audio_extractor_stage(pipeline, default_cpu_count, stage_name="audio_ext
     return stage_name
 
 
+def add_html_extractor_stage(pipeline, default_cpu_count, stage_name="html_extractor"):
+
+    pipeline.add_stage(
+        name=stage_name,
+        stage_actor=HtmlExtractorStage,
+        config=HtmlExtractorSchema(),
+        min_replicas=0,
+        max_replicas=int(max(1, (default_cpu_count // 14))),  # 7% of available CPU cores
+    )
+
+    return stage_name
+
+
 def add_otel_tracer_stage(pipeline, default_cpu_count, stage_name="otel_tracer"):
     _ = default_cpu_count  # Placeholder for future use
     otel_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317")

{nv_ingest-2025.5.21.dev20250521.dist-info → nv_ingest-2025.5.29.dev20250529.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.5.21.dev20250521
+Version: 2025.5.29.dev20250529
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License
@@ -225,13 +225,13 @@ Requires-Dist: httpx>=0.28.1
 Requires-Dist: isodate>=0.7.2
 Requires-Dist: langdetect>=1.0.9
 Requires-Dist: minio>=7.2.12
-Requires-Dist: openai>=1.57.1
+Requires-Dist: openai>=1.82.0
 Requires-Dist: opentelemetry-api>=1.27.0
 Requires-Dist: opentelemetry-exporter-otlp>=1.27.0
 Requires-Dist: opentelemetry-sdk>=1.27.0
 Requires-Dist: pydantic>2.0.0
 Requires-Dist: pydantic-settings>2.0.0
-Requires-Dist: pypdfium2>=4.30.0
+Requires-Dist: pypdfium2==4.30.1
 Requires-Dist: pytest>=8.0.2
 Requires-Dist: pytest-mock>=3.14.0
 Requires-Dist: pytest-cov>=6.0.0
@@ -239,6 +239,7 @@ Requires-Dist: build>=1.2.2
 Requires-Dist: python-docx>=1.1.2
 Requires-Dist: python-dotenv>=1.0.1
 Requires-Dist: python-pptx>=1.0.2
+Requires-Dist: prometheus-client
 Requires-Dist: torch==2.4.1
 Requires-Dist: ray[all]>=2.37.0
 Requires-Dist: redis>=5.2.1
@@ -255,9 +256,10 @@ Requires-Dist: uvicorn
 Requires-Dist: pip
 Requires-Dist: llama-index-embeddings-nvidia
 Requires-Dist: opencv-python
-Requires-Dist: pymilvus>=2.5.0
+Requires-Dist: pymilvus>=2.5.10
 Requires-Dist: pymilvus[bulk_writer,model]
 Requires-Dist: tritonclient
 Requires-Dist: nvidia-riva-client>=2.18.0
 Requires-Dist: unstructured-client
+Requires-Dist: markitdown
 Dynamic: license-file