nv-ingest 2025.8.13.dev20250813__py3-none-any.whl → 2025.8.15.dev20250815__py3-none-any.whl

nv_ingest/pipeline/pipeline_schema.py

@@ -0,0 +1,398 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from enum import Enum
+from typing import Dict, Any, List, Optional, Set, Union
+from pydantic import BaseModel, Field, field_validator, model_validator, ConfigDict
+
+from nv_ingest_api.internal.enums.common import PipelinePhase
+
+
+class StageType(str, Enum):
+    """
+    The type of a pipeline stage.
+    """
+
+    SOURCE = "source"
+    STAGE = "stage"
+    SINK = "sink"
+
+
+class ReplicaCalculationStrategy(str, Enum):
+    """
+    Strategy for calculating replica counts at runtime.
+    """
+
+    STATIC = "static"  # Fixed number of replicas
+    CPU_PERCENTAGE = "cpu_percentage"  # Percentage of available CPU cores
+    MEMORY_THRESHOLDING = "memory_thresholding"  # Based on memory allocation per replica
+    MEMORY_STATIC_GLOBAL_PERCENT = "memory_static_global_percent"  # Memory-safe calculation with budget limits
+
+
+class ReplicaStrategyConfig(BaseModel):
+    """
+    Configuration for a specific replica calculation strategy.
+
+    Attributes
+    ----------
+    strategy : ReplicaCalculationStrategy
+        The calculation strategy to use.
+    value : Optional[Union[int, float]]
+        The primary value for the strategy (e.g., static count, CPU percentage).
+    limit : Optional[int]
+        Optional upper limit for calculated replicas.
+    cpu_percent : Optional[float]
+        CPU percentage for CPU_PERCENTAGE strategy (0.0 to 1.0).
+    memory_per_replica_mb : Optional[int]
+        Expected memory usage per replica in MB.
+    memory_threshold_percent : Optional[float]
+        Memory threshold percentage for MEMORY_THRESHOLDING strategy (0.0 to 1.0).
+    max_memory_budget_mb : Optional[int]
+        Maximum memory budget for MEMORY_STATIC_GLOBAL_PERCENT strategy in MB.
+    """
+
+    strategy: ReplicaCalculationStrategy = Field(..., description="The calculation strategy to use.")
+    value: Optional[Union[int, float]] = Field(None, description="Primary value for the strategy.")
+    limit: Optional[int] = Field(None, description="Optional upper limit for calculated replicas.", ge=1)
+    cpu_percent: Optional[float] = Field(
+        None, description="CPU percentage for CPU_PERCENTAGE strategy.", ge=0.0, le=1.0
+    )
+    memory_per_replica_mb: Optional[int] = Field(None, description="Expected memory usage per replica in MB.", gt=0)
+    memory_threshold_percent: Optional[float] = Field(
+        None, description="Memory threshold percentage for MEMORY_THRESHOLDING strategy.", ge=0.0, le=1.0
+    )
+    max_memory_budget_mb: Optional[int] = Field(
+        None, description="Maximum memory budget for MEMORY_STATIC_GLOBAL_PERCENT strategy in MB.", gt=0
+    )
+
+    @model_validator(mode="after")
+    def validate_strategy_config(self):
+        """Validate that required fields are present for each strategy."""
+        if self.strategy == ReplicaCalculationStrategy.STATIC:
+            if self.value is None or not isinstance(self.value, int):
+                raise ValueError("STATIC strategy requires 'value' as an integer")
+        elif self.strategy == ReplicaCalculationStrategy.CPU_PERCENTAGE:
+            if self.cpu_percent is None:
+                if self.value is None or not isinstance(self.value, (int, float)):
+                    raise ValueError("CPU_PERCENTAGE strategy requires 'cpu_percent' or 'value' as a float")
+                self.cpu_percent = float(self.value)
+        elif self.strategy == ReplicaCalculationStrategy.MEMORY_THRESHOLDING:
+            if self.memory_per_replica_mb is None:
+                raise ValueError("MEMORY_THRESHOLDING strategy requires 'memory_per_replica_mb'")
+        elif self.strategy == ReplicaCalculationStrategy.MEMORY_STATIC_GLOBAL_PERCENT:
+            if self.memory_per_replica_mb is None:
+                raise ValueError("MEMORY_STATIC_GLOBAL_PERCENT strategy requires 'memory_per_replica_mb'")
+            # max_memory_budget_mb is optional - uses global static_memory_threshold if not provided
+        return self
+
+
+class ReplicaConfig(BaseModel):
+    """
+    Configuration for stage replicas supporting both dynamic and static scaling modes.
+
+    Defines the min/max number of replicas for a stage, either as absolute counts,
+    percentages of total CPU cores, or resource-based calculations. Supports different
+    configurations for dynamic vs static scaling modes.
+
+    Attributes
+    ----------
+    cpu_count_min : Optional[int]
+        Absolute minimum number of replicas. Must be >= 0. (Legacy support)
+    cpu_count_max : Optional[int]
+        Absolute maximum number of replicas. Must be >= 1. (Legacy support)
+    cpu_percent_min : Optional[float]
+        Minimum number of replicas as a percentage (0.0 to 1.0) of total cores. (Legacy support)
+    cpu_percent_max : Optional[float]
+        Maximum number of replicas as a percentage (0.0 to 1.0) of total cores. (Legacy support)
+    min_replicas : Optional[int]
+        Minimum number of replicas for both scaling modes. Must be >= 0.
+    max_replicas : Optional[Union[int, ReplicaStrategyConfig]]
+        Maximum replicas for dynamic scaling mode. Can be static int or strategy config.
+    static_replicas : Optional[Union[int, ReplicaStrategyConfig]]
+        Replica configuration for static scaling mode. Can be static int or strategy config.
+    """
+
+    # Legacy fields for backward compatibility
+    cpu_count_min: Optional[int] = Field(None, description="Absolute minimum number of replicas.", ge=0)
+    cpu_count_max: Optional[int] = Field(None, description="Absolute maximum number of replicas.", ge=1)
+    cpu_percent_min: Optional[float] = Field(
+        None, description="Minimum number of replicas as a percentage of total cores.", ge=0.0, le=1.0
+    )
+    cpu_percent_max: Optional[float] = Field(
+        None, description="Maximum number of replicas as a percentage of total cores.", ge=0.0, le=1.0
+    )
+
+    # New flexible replica configuration
+    min_replicas: Optional[int] = Field(None, description="Minimum number of replicas.", ge=0)
+    max_replicas: Optional[Union[int, ReplicaStrategyConfig]] = Field(
+        None, description="Maximum replicas for dynamic scaling mode."
+    )
+    static_replicas: Optional[Union[int, ReplicaStrategyConfig]] = Field(
+        None, description="Replica configuration for static scaling mode."
+    )
+
+    @model_validator(mode="after")
+    def check_exclusive_min_max(self) -> "ReplicaConfig":
+        """
+        Validates that replica configuration is consistent and complete.
+
+        Ensures that:
+        1. Legacy fields (cpu_count_*, cpu_percent_*) are not mixed with new fields
+        2. At least one configuration method is specified
+        3. Min/max relationships are valid
+        """
+        legacy_fields = [self.cpu_count_min, self.cpu_count_max, self.cpu_percent_min, self.cpu_percent_max]
+        new_fields = [self.min_replicas, self.max_replicas, self.static_replicas]
+
+        has_legacy = any(field is not None for field in legacy_fields)
+        has_new = any(field is not None for field in new_fields)
+
+        if has_legacy and has_new:
+            raise ValueError(
+                "Cannot mix legacy replica fields (cpu_count_*, cpu_percent_*) with new fields "
+                "(min_replicas, max_replicas, static_replicas). Use one approach or the other."
+            )
+
+        if not has_legacy and not has_new:
+            # Set sensible defaults for new configuration
+            self.min_replicas = 0
+            self.max_replicas = 1
+
+        # Legacy validation (existing logic)
+        if has_legacy:
+            if self.cpu_count_min is not None and self.cpu_percent_min is not None:
+                raise ValueError("Cannot specify both cpu_count_min and cpu_percent_min")
+            if self.cpu_count_max is not None and self.cpu_percent_max is not None:
+                raise ValueError("Cannot specify both cpu_count_max and cpu_percent_max")
+
+            # Validate min <= max for legacy fields
+            if self.cpu_count_min is not None and self.cpu_count_max is not None:
+                if self.cpu_count_min > self.cpu_count_max:
+                    raise ValueError("cpu_count_min cannot be greater than cpu_count_max")
+            if self.cpu_percent_min is not None and self.cpu_percent_max is not None:
+                if self.cpu_percent_min > self.cpu_percent_max:
+                    raise ValueError("cpu_percent_min cannot be greater than cpu_percent_max")
+
+        # New configuration validation
+        if has_new:
+            # Validate min_replicas against max_replicas if both are static integers
+            if (
+                self.min_replicas is not None
+                and isinstance(self.max_replicas, int)
+                and self.min_replicas > self.max_replicas
+            ):
+                raise ValueError("min_replicas cannot be greater than max_replicas")
+
+            # Validate min_replicas against static_replicas if both are static integers
+            if (
+                self.min_replicas is not None
+                and isinstance(self.static_replicas, int)
+                and self.min_replicas > self.static_replicas
+            ):
+                raise ValueError("min_replicas cannot be greater than static_replicas")
+
+        return self
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class StageConfig(BaseModel):
+    """
+    Configuration for a single pipeline stage.
+
+    Describes a single component in the ingestion pipeline, including its name,
+    type, actor implementation, and specific configuration.
+
+    Attributes
+    ----------
+    name : str
+        A unique name to identify the stage within the pipeline.
+    type : StageType
+        The type of the stage, which determines how it's added to the RayPipeline.
+    phase: PipelinePhase
+        The logical phase of the stage in the pipeline.
+    actor : Optional[str]
+        The fully qualified import path to the actor class or function that
+        implements the stage's logic. Mutually exclusive with 'callable'.
+    callable : Optional[str]
+        The fully qualified import path to a callable function that
+        implements the stage's logic. Mutually exclusive with 'actor'.
+    task_filters: Optional[List[Any]]
+        List of task types this callable stage should filter for. Only applies to callable stages.
+        Supports both simple strings (e.g., "udf") and complex filters (e.g., ["udf", {"phase": 5}]).
+    enabled : bool
+        A flag to indicate whether the stage should be included in the pipeline.
+        If False, the stage and its connected edges are ignored.
+    config : Dict[str, Any]
+        A dictionary of configuration parameters passed to the stage's actor.
+    replicas : ReplicaConfig
+        The replica configuration for the stage.
+    runs_after: List[str]
+        A list of stage names that this stage must be downstream of.
+    """
+
+    name: str = Field(..., description="Unique name for the stage.")
+    type: StageType = Field(StageType.STAGE, description="Type of the stage.")
+    phase: PipelinePhase = Field(..., description="The logical phase of the stage.")
+    actor: Optional[str] = Field(None, description="Full import path to the stage's actor class or function.")
+    callable: Optional[str] = Field(None, description="Full import path to a callable function for the stage.")
+    task_filters: Optional[List[Any]] = Field(
+        None, description="List of task types this callable stage should filter for. Only applies to callable stages."
+    )
+    enabled: bool = Field(True, description="Whether the stage is enabled.")
+    config: Dict[str, Any] = Field({}, description="Configuration dictionary for the stage.")
+    replicas: ReplicaConfig = Field(default_factory=ReplicaConfig, description="Replica configuration.")
+    runs_after: List[str] = Field(default_factory=list, description="List of stages this stage must run after.")
+
+    @model_validator(mode="after")
+    def check_actor_or_callable(self) -> "StageConfig":
+        """
+        Validates that exactly one of 'actor' or 'callable' is specified.
+        """
+        if self.actor is None and self.callable is None:
+            raise ValueError("Either 'actor' or 'callable' must be specified")
+
+        if self.actor is not None and self.callable is not None:
+            raise ValueError("Cannot specify both 'actor' and 'callable' - they are mutually exclusive")
+
+        return self
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class EdgeConfig(BaseModel):
+    """
+    Configuration for an edge between two stages.
+
+    Defines a connection from a source stage to a destination stage, including
+    the size of the intermediate queue.
+
+    Attributes
+    ----------
+    from_stage : str
+        The name of the source stage for the edge.
+    to_stage : str
+        The name of the destination stage for the edge.
+    queue_size : int
+        The maximum number of items in the queue between the two stages.
+    """
+
+    from_stage: str = Field(..., alias="from", description="The name of the source stage.")
+    to_stage: str = Field(..., alias="to", description="The name of the destination stage.")
+    queue_size: int = Field(100, gt=0, description="The size of the queue between stages.")
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PIDControllerConfig(BaseModel):
+    """
+    Configuration for the PID controller used in dynamic scaling.
+
+    Attributes
+    ----------
+    kp : float
+        Proportional gain for the PID controller.
+    ki : float
+        Integral gain for the PID controller.
+    ema_alpha : float
+        Exponential moving average alpha for the PID controller.
+    target_queue_depth : int
+        Target queue depth for the PID controller.
+    penalty_factor : float
+        Penalty factor for the PID controller.
+    error_boost_factor : float
+        Error boost factor for the PID controller.
+    rcm_memory_safety_buffer_fraction : float
+        Resource constraint manager memory safety buffer fraction.
+    """
+
+    kp: float = Field(0.2, gt=0.0, description="Proportional gain for the PID controller.")
+    ki: float = Field(0.01, ge=0.0, description="Integral gain for the PID controller.")
+    ema_alpha: float = Field(
+        0.1, ge=0.0, le=1.0, description="Exponential moving average alpha for the PID controller."
+    )
+    target_queue_depth: int = Field(0, ge=0, description="Target queue depth for the PID controller.")
+    penalty_factor: float = Field(0.1, ge=0.0, description="Penalty factor for the PID controller.")
+    error_boost_factor: float = Field(1.5, gt=0.0, description="Error boost factor for the PID controller.")
+    rcm_memory_safety_buffer_fraction: float = Field(
+        0.15, ge=0.0, le=1.0, description="Resource constraint manager memory safety buffer fraction."
+    )
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PipelineRuntimeConfig(BaseModel):
+    """
+    Configuration for pipeline runtime behavior.
+
+    Parameters
+    ----------
+    disable_dynamic_scaling : bool
+        Whether to disable dynamic scaling of replicas (default: False).
+    dynamic_memory_threshold : float
+        The memory utilization threshold (0.0 to 1.0) for dynamic scaling decisions.
+    static_memory_threshold : float
+        Global memory threshold for static scaling mode (default: 0.75).
+    pid_controller : PIDControllerConfig
+        PID controller configuration for dynamic scaling.
+    launch_simple_broker : bool
+        If True, launches a simple message broker for the pipeline.
+    """
+
+    disable_dynamic_scaling: bool = Field(False, description="Disable dynamic scaling of stage replicas.")
+    dynamic_memory_threshold: float = Field(
+        0.75, ge=0.0, le=0.95, description="Memory utilization threshold for dynamic scaling."
+    )
+    static_memory_threshold: float = Field(
+        0.75, ge=0.0, le=1.0, description="Global memory threshold for static scaling mode."
+    )
+    pid_controller: PIDControllerConfig = Field(
+        default_factory=PIDControllerConfig, description="PID controller configuration for dynamic scaling."
+    )
+    launch_simple_broker: bool = Field(False, description="Launch a simple message broker for the pipeline.")
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PipelineConfigSchema(BaseModel):
+    """
+    Root configuration model for an ingestion pipeline.
+
+    This model represents the entire declarative configuration for an ingestion
+    pipeline, including all stages and the edges that connect them.
+
+    Attributes
+    ----------
+    name : str
+        The name of the pipeline.
+    description : str
+        A description of the pipeline.
+    stages : List[StageConfig]
+        A list of all stage configurations in the pipeline.
+    edges : List[EdgeConfig]
+        A list of all edge configurations that define the pipeline's topology.
+    pipeline: Optional[PipelineRuntimeConfig] = Field(default_factory=PipelineRuntimeConfig,
+        description="Runtime configuration for the pipeline.")
+    """
+
+    name: str = Field(..., description="The name of the pipeline.")
+    description: str = Field(..., description="A description of the pipeline.")
+    stages: List[StageConfig] = Field(..., description="List of all stages in the pipeline.")
+    edges: List[EdgeConfig] = Field(..., description="List of all edges connecting the stages.")
+    pipeline: Optional[PipelineRuntimeConfig] = Field(
+        default_factory=PipelineRuntimeConfig, description="Runtime configuration for the pipeline."
+    )
+
+    @field_validator("stages", "edges")
+    def check_not_empty(cls, v: list) -> list:
+        """Validates that the list is not empty."""
+        if not v:
+            raise ValueError("must not be empty")
+        return v
+
+    def get_phases(self) -> Set[PipelinePhase]:
+        """Returns a set of all unique phases in the pipeline."""
+        return {stage.phase for stage in self.stages}
+
+    model_config = ConfigDict(extra="forbid")

{nv_ingest-2025.8.13.dev20250813.dist-info → nv_ingest-2025.8.15.dev20250815.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 2025.8.13.dev20250813
+Version: 2025.8.15.dev20250815
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest-2025.8.13.dev20250813.dist-info → nv_ingest-2025.8.15.dev20250815.dist-info}/RECORD

@@ -8,68 +8,74 @@ nv_ingest/api/v1/ingest.py,sha256=LWk3LN4lBd3uO8h30EN42g3LHCVcO00avVd5ohVK7NI,19
 nv_ingest/api/v1/metrics.py,sha256=ZGVRApYLnzc2f2C7wRgGd7deqiXan-jxfA-33a16clY,981
 nv_ingest/framework/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
+nv_ingest/framework/orchestration/execution/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
+nv_ingest/framework/orchestration/execution/helpers.py,sha256=-F8SZh7ISWtzJz6X1O2LQ133t-17Jxi8lL-NHz4rwj0,2818
+nv_ingest/framework/orchestration/execution/options.py,sha256=Ms1t4591EIv4ZrMRdhsCYPgLnMVXJosG3MURCbPXUoA,3983
+nv_ingest/framework/orchestration/process/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
+nv_ingest/framework/orchestration/process/dependent_services.py,sha256=GpYYQ-YPKJbAFTcMHk4wCQUvtkJCfMOqCss5j845x0g,1979
+nv_ingest/framework/orchestration/process/execution.py,sha256=YkeNKk2H7DpiF2-9ygjKH2DeIM7dpvS4a74gDVBC0Eo,19540
+nv_ingest/framework/orchestration/process/lifecycle.py,sha256=xM6zjMJfoI1KyFlr76fcE9N8GINV3oG_5n1aw4P7AlY,4160
+nv_ingest/framework/orchestration/process/strategies.py,sha256=LzJDx9g_QdHtfBzg-edNMSqdYRZfF71i9S3Fv1ghK_M,6377
 nv_ingest/framework/orchestration/ray/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/ray/edges/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/ray/edges/async_queue_edge.py,sha256=PQliU_kyGbO9o42njpb8FrDMLrbLqwZzmBNXifxyG5Y,2312
 nv_ingest/framework/orchestration/ray/edges/ray_queue_edge.py,sha256=VFii2yxJuikimOxie3edKq5JN06g78AF8bdHSHVX8p8,2677
 nv_ingest/framework/orchestration/ray/edges/threaded_queue_edge.py,sha256=N6NH4KgZJ60e_JkGRcSmfQtX37qtX4TMcavOR-n3heE,2549
 nv_ingest/framework/orchestration/ray/examples/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/framework/orchestration/ray/examples/pipeline_test_harness.py,sha256=DufjmNm-05uTkq_Mz0QQB6fHw_Rl9eX3PRtnH4sntGs,16405
+nv_ingest/framework/orchestration/ray/examples/pipeline_test_harness.py,sha256=hnRLybIpVTj3mXkLW0ErWVn4vRsInjNZmA80JqDiQuw,16473
 nv_ingest/framework/orchestration/ray/examples/task_source_harness.py,sha256=Yt7uxThg7s8WuMiaHLKC8r1XAG7QixegfkT-juE5oNw,1953
 nv_ingest/framework/orchestration/ray/examples/task_source_sink_harness.py,sha256=XkvsoIzH5ftXvAZ4ox7mxbx7ESVx6D8Xupcwbqgd52w,3277
 nv_ingest/framework/orchestration/ray/primitives/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/ray/primitives/dataclasses.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nv_ingest/framework/orchestration/ray/primitives/pipeline_monitor.py,sha256=L8ENPiF-lxqhIXVEQwQD5CCqQMb710ynj5D_Y4ixGhs,11077
-nv_ingest/framework/orchestration/ray/primitives/pipeline_topology.py,sha256=IxLQYHYc9BnMOi73TSJzuhl8KOJAwlwwp8SPdUkV3nE,15737
-nv_ingest/framework/orchestration/ray/primitives/ray_pipeline.py,sha256=HGJ_TyLTKKRl10HWfyx3D-n-zrFY0Fg9TN74UbOeCm8,66584
-nv_ingest/framework/orchestration/ray/primitives/ray_stat_collector.py,sha256=8SpZzulHatqah7U3YHJMTLaYyPlWdCoaer_oNjhmHZo,17221
+nv_ingest/framework/orchestration/ray/primitives/pipeline_topology.py,sha256=yd2sb_q_FbBLDWiYgdKRhpPVAIl26Wg8w3yelZ7c5zQ,15741
+nv_ingest/framework/orchestration/ray/primitives/ray_pipeline.py,sha256=ncVOYTCrvqcipUdLBmBgH5vKpGTRrX6MrgGyBNPVknM,66656
+nv_ingest/framework/orchestration/ray/primitives/ray_stat_collector.py,sha256=GGY6_i6_g5xTFzdo9Qmsu9i4knMTq6pJfgm-aaPEt_o,17226
 nv_ingest/framework/orchestration/ray/stages/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/ray/stages/extractors/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/framework/orchestration/ray/stages/extractors/audio_extractor.py,sha256=KV4hvY0NTGG8CjZviTgcFLQzaH8WJJGkkb9PFYbROww,3417
-nv_ingest/framework/orchestration/ray/stages/extractors/chart_extractor.py,sha256=tydluNNXfZYSo-0eqqafB59icF3SaeLXWcMrZ6OzlyQ,3998
-nv_ingest/framework/orchestration/ray/stages/extractors/docx_extractor.py,sha256=tSa3Z4vK6sYJ6RBNMa7_FiuOwUaDUl0rTJ6agGbI5y0,3426
-nv_ingest/framework/orchestration/ray/stages/extractors/html_extractor.py,sha256=fyr0oXokhuaGQrNu5rKyH_qNMD12AS1xPDxKgA26YHE,3426
-nv_ingest/framework/orchestration/ray/stages/extractors/image_extractor.py,sha256=c-qlLGSizLOgKqH7wl_c8dGOVKYxLtXhZEHLXil4Jc4,3734
-nv_ingest/framework/orchestration/ray/stages/extractors/infographic_extractor.py,sha256=dmgvzGMxVX81g7TpZO1ACnRh7sdtpc7YX5KK2QW26U4,2565
-nv_ingest/framework/orchestration/ray/stages/extractors/pdf_extractor.py,sha256=BUVuYOCGyPdPpacVhL5rnvA56hydnBip7tPaWTXaT1c,4650
-nv_ingest/framework/orchestration/ray/stages/extractors/pptx_extractor.py,sha256=ywPGA-3GNsbp3FWFsu04foumM6ZCccRrm73ijS7oY0g,3581
-nv_ingest/framework/orchestration/ray/stages/extractors/table_extractor.py,sha256=EOcjyJYAB3TuXewZFld4shnGQUQ9VysjPrIWnmb8zuI,3893
+nv_ingest/framework/orchestration/ray/stages/extractors/audio_extractor.py,sha256=MdJA79_XECTmvksmxRZiIj5cZ46HzvIYOFNi5-S9o1k,3540
+nv_ingest/framework/orchestration/ray/stages/extractors/chart_extractor.py,sha256=D_clXtJe63CkeOcqor6C-X3Gbny7rNlp8PRMp9sRuPw,4281
+nv_ingest/framework/orchestration/ray/stages/extractors/docx_extractor.py,sha256=8DQOivEfvptUY7x7I6NsdJMTkiLdq2e_xPpPl3_bLJY,3536
+nv_ingest/framework/orchestration/ray/stages/extractors/html_extractor.py,sha256=fp0nzRQ21PmIInxtXjTYhOiYytmgBypDlY0e-WHCmGA,3551
+nv_ingest/framework/orchestration/ray/stages/extractors/image_extractor.py,sha256=CMIfHLCE4DUIedTxedZWWJKWlrLbWZzsuEE3j_ix_nY,3889
+nv_ingest/framework/orchestration/ray/stages/extractors/infographic_extractor.py,sha256=v5J7dnJBEaDfjoTz_N_yC3RAt6lwMLgLT28V-ahquLE,3261
+nv_ingest/framework/orchestration/ray/stages/extractors/pdf_extractor.py,sha256=5gpbf2gJiUmL0-XR6nxWih3E0_FqkMRDaLdgP_MQs-c,4892
+nv_ingest/framework/orchestration/ray/stages/extractors/pptx_extractor.py,sha256=RMbbl7Cuj4BT-TcgUx_0k8R-DLdw-o3fHxcIBIgrWt4,3776
+nv_ingest/framework/orchestration/ray/stages/extractors/table_extractor.py,sha256=YvCsBD4hxzgY-vGQmDwXVhleClMj2PHzsX71I5_VM_o,4079
 nv_ingest/framework/orchestration/ray/stages/injectors/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/framework/orchestration/ray/stages/injectors/metadata_injector.py,sha256=K8jase7PD9kd8AuntzjdS1IO4ae8Oo_6byZsFG777D0,6838
+nv_ingest/framework/orchestration/ray/stages/injectors/metadata_injector.py,sha256=b4yve-J7ZMURSSuOPzSvfdIcyZ9fX8d-q7cYnXuxgsM,7025
 nv_ingest/framework/orchestration/ray/stages/meta/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/ray/stages/meta/ray_actor_edge_base.py,sha256=LnVqBJmpfCmcI-eJLbkwK-7SS-hpEp98P4iCRv_Zhb0,1726
-nv_ingest/framework/orchestration/ray/stages/meta/ray_actor_sink_stage_base.py,sha256=AhlZUbDK2Jckqnu8hVbJrckW8MsSixfmWc1bst9gRYk,3447
-nv_ingest/framework/orchestration/ray/stages/meta/ray_actor_source_stage_base.py,sha256=1Pae2xRPK0_QLh53yHECVFm2guwgvZaiRRr3tp4OpYI,1744
-nv_ingest/framework/orchestration/ray/stages/meta/ray_actor_stage_base.py,sha256=B1T-NvpI5N4qmawoEbNJf7fpI04ZVE-1tNRcV31Mkuw,29459
+nv_ingest/framework/orchestration/ray/stages/meta/ray_actor_sink_stage_base.py,sha256=HQJXIuU7VjiQ6fQjHjbNNmIJX5f30cXFB0CJGixgwVo,3633
+nv_ingest/framework/orchestration/ray/stages/meta/ray_actor_source_stage_base.py,sha256=hP25MLTP2bOEEncrYdxPPqeRyRVbij8aEurR1F1ZmhE,1811
+nv_ingest/framework/orchestration/ray/stages/meta/ray_actor_stage_base.py,sha256=qiB_ZU5_3bXgvE9C2rvnXIS0Alm6M5PWLCeQm8ZxOy4,29812
 nv_ingest/framework/orchestration/ray/stages/mutate/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/framework/orchestration/ray/stages/mutate/image_dedup.py,sha256=UepeDvH6Cfgm5rIylRx6uOxihS0OZ4Q1DGUrjUybNaY,3493
-nv_ingest/framework/orchestration/ray/stages/mutate/image_filter.py,sha256=9ek5rVa4_GVdmVHGMJvbxacRSpIqVoUxgv28lzJwrTQ,3319
+nv_ingest/framework/orchestration/ray/stages/mutate/image_dedup.py,sha256=ipH-3dctKaXRbjfkcXfpKRPXJBHXEosqPpm_Gb-5PZ0,3618
+nv_ingest/framework/orchestration/ray/stages/mutate/image_filter.py,sha256=-UkLArpgP1cr433dQDzO0g5Wn8cpvjqK6RGamufO5gQ,3432
 nv_ingest/framework/orchestration/ray/stages/sinks/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/framework/orchestration/ray/stages/sinks/default_drain.py,sha256=0SQHJlFuXlP16YRWduX1fMKgjhUd7UhDAWQ8XZh4_0I,1471
-nv_ingest/framework/orchestration/ray/stages/sinks/message_broker_task_sink.py,sha256=enylryvcPmzirpOjCahqYJbNSLsNvv1KpMnOzGqNZQQ,11509
+nv_ingest/framework/orchestration/ray/stages/sinks/default_drain.py,sha256=_USW1Vq8G2Wn-QFdPfFQCrtKG46hHeJvkEGbBxdpbVM,1488
+nv_ingest/framework/orchestration/ray/stages/sinks/message_broker_task_sink.py,sha256=QcvMQXIJ7EWIxty76Mo5Xv38Oj6X2KuS8qXQlf7E1uA,11676
 nv_ingest/framework/orchestration/ray/stages/sources/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/framework/orchestration/ray/stages/sources/message_broker_task_source.py,sha256=b9ndnQBB1paR0iRe3NdzQ7BZ2S65LG2jbtjXvvDc_s4,21183
+nv_ingest/framework/orchestration/ray/stages/sources/message_broker_task_source.py,sha256=A_D17mOqTh1a-4_FbcEUzPOhCxK7gqrXdEHXHpnHTcU,22163
 nv_ingest/framework/orchestration/ray/stages/storage/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/framework/orchestration/ray/stages/storage/image_storage.py,sha256=6NkwQzseAnaj0Ptpr3oKvab2EnJdMwTjI2p4dS_HzsI,3901
-nv_ingest/framework/orchestration/ray/stages/storage/store_embeddings.py,sha256=SMLHQElZkKldnjy0_VHIKS65DBAAtOhwhdoaFe1yb9I,3337
+nv_ingest/framework/orchestration/ray/stages/storage/image_storage.py,sha256=WZN_-3Li-izDaPtk8IMrtn2os1ckT3U8Rb2PsfOWrcI,4009
+nv_ingest/framework/orchestration/ray/stages/storage/store_embeddings.py,sha256=4-bV4lIAdeylBeBHMO-6bVa2xilhPdZqkwfM8OwI2rM,3456
 nv_ingest/framework/orchestration/ray/stages/telemetry/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/framework/orchestration/ray/stages/telemetry/job_counter.py,sha256=a6-GKP9w02DsMsZk6Fi-MLTRCW2MI7dnO1N2JCQFUIo,2656
+nv_ingest/framework/orchestration/ray/stages/telemetry/job_counter.py,sha256=jEtEUibqs6IS6QakrzWY9zmxSUzuBpg_hzXy2R-I10Y,2870
 nv_ingest/framework/orchestration/ray/stages/telemetry/otel_meter.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/framework/orchestration/ray/stages/telemetry/otel_tracer.py,sha256=53M8Xw84gcuRAeHbf4Z_ApLFX9Zkd1KSec_2k-wXl6c,7947
+nv_ingest/framework/orchestration/ray/stages/telemetry/otel_tracer.py,sha256=GqFwRpTlIu2qgb08B3OqVIKUG8QQAwSOHhjvnyL_9UE,8553
 nv_ingest/framework/orchestration/ray/stages/transforms/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/framework/orchestration/ray/stages/transforms/image_caption.py,sha256=OuqPmJmCqbg9k7roDivuvfYVTd05Nl9PMC0_E9PHgYw,3514
-nv_ingest/framework/orchestration/ray/stages/transforms/text_embed.py,sha256=urXv7fj17DBY5YLwxCkSyUAROjANTFCYo_EOlBYXlMg,3293
-nv_ingest/framework/orchestration/ray/stages/transforms/text_splitter.py,sha256=hyTIv3MMnuyZmgbBHbXicSgk9DhbpygDrgUtXRwvlmo,4677
+nv_ingest/framework/orchestration/ray/stages/transforms/image_caption.py,sha256=Ghg09z5fSH0uyGplJPybE1e6sCD9IFpkGjzU3x8zd9o,3616
+nv_ingest/framework/orchestration/ray/stages/transforms/text_embed.py,sha256=RT4SkyjHPSJS7TuQhIcNW3B5a2tahEEtlxwB7M2UBYk,3309
+nv_ingest/framework/orchestration/ray/stages/transforms/text_splitter.py,sha256=D8HGPwOCqgCA6PMoeBAHaxKfLUgPhTlSP84uiujskYM,4829
 nv_ingest/framework/orchestration/ray/stages/utility/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/framework/orchestration/ray/stages/utility/throughput_monitor.py,sha256=MB27CkoNeuirN6CUHgjsC5Wh958NF7m_N7HE4VKfx3k,2264
+nv_ingest/framework/orchestration/ray/stages/utility/throughput_monitor.py,sha256=J7Qs745rt7cQlR3L2K9U6Mb_BDKUNpl_xoqCZMEPlks,2674
 nv_ingest/framework/orchestration/ray/util/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/ray/util/env_config.py,sha256=GN9msJ_3jdOBIAPnXNxX0ds_BKtHRnRhnYxwzcAU2KY,2386
 nv_ingest/framework/orchestration/ray/util/pipeline/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest/framework/orchestration/ray/util/pipeline/pid_controller.py,sha256=flRLS7yc5n6gheykayuL3prC7O-ZhcVY2s9Wc14SGWE,47377
-nv_ingest/framework/orchestration/ray/util/pipeline/pipeline_builders.py,sha256=cZhx3jamrwyU3v_LYwNlsGa9AuGQeI9aiEKbijuKa4o,17558
-nv_ingest/framework/orchestration/ray/util/pipeline/pipeline_runners.py,sha256=-lXiAjxpk3CxIFWi7GW8ggTn1cjyugYZ9EtSfTT20mg,14844
-nv_ingest/framework/orchestration/ray/util/pipeline/stage_builders.py,sha256=CD5pAa8oQJVplCr2snzZXJyRjSclcIPGw8aFPI2dU1k,22246
-nv_ingest/framework/orchestration/ray/util/pipeline/tools.py,sha256=LQVb8k9jURaxh2Ga44Js_XuYFCbeN4_nLgDmtExovQg,8026
+nv_ingest/framework/orchestration/ray/util/pipeline/pid_controller.py,sha256=0dSDVTv3FXjMZ79sQh4i4YEwnqND5iPw8GAeZI0oJO4,47338
+nv_ingest/framework/orchestration/ray/util/pipeline/pipeline_runners.py,sha256=zWi-6-7dfb_3R00uVi3wdYMH1HgeevkBkg47UY8QqUQ,4386
+nv_ingest/framework/orchestration/ray/util/pipeline/tools.py,sha256=MzxLjElEVb6C5ghfJ7GCp8uqNZeVuzz8xJnxzdQmOsI,8425
 nv_ingest/framework/orchestration/ray/util/system_tools/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/orchestration/ray/util/system_tools/memory.py,sha256=ICqY0LLB3hFTZk03iX5yffMSKFH2q_aQomtDVzS_mKw,2228
 nv_ingest/framework/orchestration/ray/util/system_tools/visualizers.py,sha256=2oHZdO_3L1LGuzpyNmZBDh19n0E-APAaHk4MEwBwSHs,12895
@@ -88,6 +94,7 @@ nv_ingest/framework/schemas/framework_vdb_task_sink_schema.py,sha256=gcd27IM2R2Y
 nv_ingest/framework/util/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/util/flow_control/__init__.py,sha256=blU7s6_dxTG2gHo6YadXslT0J-gkdl6dcCADZgpnjGc,215
 nv_ingest/framework/util/flow_control/filter_by_task.py,sha256=EMGoNNPiDbfVDGzXfGAxoxllqhFPqRkh9VqCidxyibI,10893
+nv_ingest/framework/util/flow_control/udf_intercept.py,sha256=zQ9uuCcHLEd0P52EiwQB9hY-90X1HashVeyHanZJ3sg,14452
 nv_ingest/framework/util/service/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/util/service/impl/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/util/service/impl/ingest/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
@@ -97,8 +104,15 @@ nv_ingest/framework/util/service/meta/ingest/__init__.py,sha256=wQSlVx3T14ZgQAt-
 nv_ingest/framework/util/service/meta/ingest/ingest_service_meta.py,sha256=QS3uNxWBl5dIcmIpJKNe8_TLcTUuN2vcKyHeAwa-eSo,1589
 nv_ingest/framework/util/telemetry/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest/framework/util/telemetry/global_stats.py,sha256=nq65pEEdiwjAfGiqsxG1CeQMC96O3CfQxsZuGFCY-ds,4554
-nv_ingest-2025.8.13.dev20250813.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-nv_ingest-2025.8.13.dev20250813.dist-info/METADATA,sha256=KT9H2WDuNXNMaboxtMApQayN4Zt6KjBBdMDjEbecB3k,15061
-nv_ingest-2025.8.13.dev20250813.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nv_ingest-2025.8.13.dev20250813.dist-info/top_level.txt,sha256=sjb0ajIsgn3YgftSjZHlYO0HjYAIIhNuXG_AmywCvaU,10
-nv_ingest-2025.8.13.dev20250813.dist-info/RECORD,,
+nv_ingest/pipeline/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
+nv_ingest/pipeline/default_pipeline_impl.py,sha256=TV3WvmQL7DulE54rbF5-apKboVagqI-xqPFx24Sq_8Q,15582
+nv_ingest/pipeline/ingest_pipeline.py,sha256=wHAJhqAM2s8nbY-8itVogmSU-yVN4PZONGWcKnhzgfg,17794
+nv_ingest/pipeline/pipeline_schema.py,sha256=rLZZz2It2o2hVNWrZUJU8CarrqRei1fho3ZEMkkoBcg,17940
+nv_ingest/pipeline/config/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
+nv_ingest/pipeline/config/loaders.py,sha256=Zt9QOEd6g59pyN0zJZjh4VfXpjCHg2AR_4Wsbu4GkbM,6599
+nv_ingest/pipeline/config/replica_resolver.py,sha256=3zjh8gmepEYORFZRM4inq7GoBW0YL3gzUDiixUugjzQ,8899
+nv_ingest-2025.8.15.dev20250815.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+nv_ingest-2025.8.15.dev20250815.dist-info/METADATA,sha256=47V1FPaR2Fqy74TQJihwEWK6g7hzbVeNptdaV-sI6hE,15061
+nv_ingest-2025.8.15.dev20250815.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nv_ingest-2025.8.15.dev20250815.dist-info/top_level.txt,sha256=sjb0ajIsgn3YgftSjZHlYO0HjYAIIhNuXG_AmywCvaU,10
+nv_ingest-2025.8.15.dev20250815.dist-info/RECORD,,