nv-ingest 25.6.26.dev20250626__tar.gz → 25.6.28.dev20250628__tar.gz

{nv_ingest-25.6.26.dev20250626 → nv_ingest-25.6.28.dev20250628}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 25.6.26.dev20250626
+Version: 25.6.28.dev20250628
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest-25.6.26.dev20250626 → nv_ingest-25.6.28.dev20250628}/nv_ingest/framework/orchestration/ray/primitives/ray_pipeline.py

@@ -9,6 +9,7 @@ import threading
 from abc import ABC, abstractmethod
 from collections import defaultdict
 from dataclasses import dataclass
+from types import FunctionType
 
 import psutil
 import uuid
@@ -24,6 +25,9 @@ import time
 from nv_ingest.framework.orchestration.ray.primitives.pipeline_topology import PipelineTopology, StageInfo
 from nv_ingest.framework.orchestration.ray.primitives.ray_stat_collector import RayStatsCollector
 from nv_ingest.framework.orchestration.ray.util.pipeline.pid_controller import PIDController, ResourceConstraintManager
+from nv_ingest.framework.orchestration.ray.util.pipeline.tools import wrap_callable_as_stage
+from nv_ingest_api.util.imports.callable_signatures import ingest_stage_callable_signature
+from nv_ingest_api.util.imports.dynamic_resolvers import resolve_callable_from_path
 
 logger = logging.getLogger(__name__)
 
@@ -43,7 +47,7 @@ class PipelineInterface(ABC):
         Parameters
         ----------
         monitor_poll_interval : float
-            Interval in seconds for monitoring poll (default: 5.0).
+            Interval in seconds for the monitoring poll (default: 5.0).
         scaling_poll_interval : float
             Interval in seconds for scaling decisions (default: 30.0).
         """
@@ -270,7 +274,7 @@ class RayPipeline(PipelineInterface):
 
         logger.info("RayStatsCollector initialized using StatsConfig.")
 
-    # --- Accessor Methods for Stats Collector (and internal use) ---
+    # --- Accessor Methods for Stat Collector (and internal use) ---
 
     def __del__(self):
         try:
@@ -428,15 +432,39 @@ class RayPipeline(PipelineInterface):
         return self
 
     def add_stage(
-        self, *, name: str, stage_actor: Any, config: BaseModel, min_replicas: int = 0, max_replicas: int = 1
+        self,
+        *,
+        name: str,
+        stage_actor: Any,
+        config: BaseModel,
+        min_replicas: int = 0,
+        max_replicas: int = 1,
     ) -> "RayPipeline":
         if min_replicas < 0:
             logger.warning(f"Stage '{name}': min_replicas cannot be negative. Overriding to 0.")
             min_replicas = 0
+
+        resolved_actor = stage_actor
+
+        # Support module path (e.g., "mypkg.mymodule:my_lambda")
+        if isinstance(stage_actor, str):
+            resolved_actor = resolve_callable_from_path(
+                callable_path=stage_actor, signature_schema=ingest_stage_callable_signature
+            )
+
+        # Wrap callables
+        if isinstance(resolved_actor, FunctionType):
+            schema_type = type(config)
+            resolved_actor = wrap_callable_as_stage(resolved_actor, schema_type)
+
         stage_info = StageInfo(
-            name=name, callable=stage_actor, config=config, min_replicas=min_replicas, max_replicas=max_replicas
+            name=name,
+            callable=resolved_actor,
+            config=config,
+            min_replicas=min_replicas,
+            max_replicas=max_replicas,
         )
-        self.topology.add_stage(stage_info)  # Delegate
+        self.topology.add_stage(stage_info)
 
         return self
 

{nv_ingest-25.6.26.dev20250626 → nv_ingest-25.6.28.dev20250628}/nv_ingest/framework/orchestration/ray/stages/meta/ray_actor_source_stage_base.py

@@ -23,16 +23,13 @@ class RayActorSourceStage(RayActorStage, ABC):
         super().__init__(config, log_to_stdout=log_to_stdout)
         self.paused = False
 
+    def on_data(self, IngestControlMessage):
+        return NotImplemented("Source stages do not implement on_data().")
+
     @ray.method(num_returns=1)
     def set_input_queue(self, queue_handle: Any) -> bool:
         raise NotImplementedError("Source stages do not support an input queue.")
 
-    def get_input(self) -> Any:
-        """
-        Source stages must implement get_input() to fetch control messages from an external source.
-        """
-        pass
-
     @abstractmethod
     def _read_input(self) -> Any:
         """

{nv_ingest-25.6.26.dev20250626 → nv_ingest-25.6.28.dev20250628}/nv_ingest/framework/orchestration/ray/stages/sources/message_broker_task_source.py

@@ -304,14 +304,6 @@ class MessageBrokerTaskSourceStage(RayActorSourceStage):
 
         return control_message
 
-    def on_data(self, control_message: any) -> any:
-        """
-        Process the control message.
-        For this source stage, no additional processing is done, so simply return it.
-        """
-        self._logger.debug("on_data: Received control message for processing")
-        return control_message
-
     # In the processing loop, instead of checking a boolean, we wait on the event.
     def _processing_loop(self) -> None:
         """
@@ -336,7 +328,6 @@ class MessageBrokerTaskSourceStage(RayActorSourceStage):
                 self._active_processing = True
 
                 self._logger.debug("Control message received; processing data")
-                updated_cm = self.on_data(control_message)
 
                 # Block until not paused using the pause event.
                 if self.output_queue is not None:
@@ -349,7 +340,7 @@ class MessageBrokerTaskSourceStage(RayActorSourceStage):
 
                     while True:
                         try:
-                            self.output_queue.put(updated_cm)
+                            self.output_queue.put(control_message)
                             self.stats["successful_queue_writes"] += 1
                             break
                         except Exception:

{nv_ingest-25.6.26.dev20250626 → nv_ingest-25.6.28.dev20250628}/nv_ingest/framework/orchestration/ray/stages/transforms/text_splitter.py

@@ -1,11 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 import logging
 from typing import Any
 import ray
 
-# Assume these imports come from your project:
 from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
 from nv_ingest.framework.util.flow_control import filter_by_task
-from nv_ingest_api.internal.primitives.ingest_control_message import remove_task_by_type
+from nv_ingest_api.internal.primitives.ingest_control_message import remove_task_by_type, IngestControlMessage
 from nv_ingest_api.internal.primitives.tracing.tagging import traceable
 from nv_ingest_api.internal.schemas.transform.transform_text_splitter_schema import TextSplitterSchema
 from nv_ingest_api.internal.transform.split_text import transform_text_split_and_tokenize_internal
@@ -72,3 +75,45 @@ class TextSplitterStage(RayActorStage):
         logger.info("TextSplitterStage.on_data: Finished processing, returning updated message.")
 
         return message
+
+
+def text_splitter_fn(control_message: IngestControlMessage, stage_config: TextSplitterSchema) -> IngestControlMessage:
+    """
+    Process an incoming IngestControlMessage by splitting and tokenizing its text.
+
+    Parameters
+    ----------
+    control_message : IngestControlMessage
+        The incoming message containing the payload DataFrame.
+
+    stage_config : BaseModel
+        The stage level configuration object
+
+    Returns
+    -------
+    IngestControlMessage
+        The updated message with its payload transformed.
+    """
+
+    # Extract the DataFrame payload.
+    df_payload = control_message.payload()
+    logger.debug("Extracted payload with %d rows.", len(df_payload))
+
+    # Remove the "split" task to obtain task-specific configuration.
+    task_config = remove_task_by_type(control_message, "split")
+    logger.debug("Extracted task config: %s", task_config)
+
+    # Transform the DataFrame (split text and tokenize).
+    df_updated = transform_text_split_and_tokenize_internal(
+        df_transform_ledger=df_payload,
+        task_config=task_config,
+        transform_config=stage_config,
+        execution_trace_log=None,
+    )
+    logger.info("TextSplitterStage.on_data: Transformation complete. Updated payload has %d rows.", len(df_updated))
+
+    # Update the message payload.
+    control_message.payload(df_updated)
+    logger.info("TextSplitterStage.on_data: Finished processing, returning updated message.")
+
+    return control_message

{nv_ingest-25.6.26.dev20250626 → nv_ingest-25.6.28.dev20250628}/nv_ingest/framework/orchestration/ray/util/pipeline/stage_builders.py

@@ -2,8 +2,6 @@
 # All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-# TODO(Devin)
-# flake8: noqa
 import os
 
 import click
@@ -11,6 +9,7 @@ import logging
 
 from nv_ingest.framework.orchestration.ray.stages.sinks.default_drain import DefaultDrainSink
 from nv_ingest.framework.orchestration.ray.stages.telemetry.otel_tracer import OpenTelemetryTracerStage
+from nv_ingest.framework.orchestration.ray.stages.transforms.text_splitter import TextSplitterStage
 from nv_ingest.framework.schemas.framework_otel_tracer_schema import OpenTelemetryTracerSchema
 from nv_ingest_api.internal.schemas.extract.extract_infographic_schema import InfographicExtractorSchema
 
@@ -41,7 +40,6 @@ from nv_ingest.framework.orchestration.ray.stages.storage.image_storage import I
 from nv_ingest.framework.orchestration.ray.stages.storage.store_embeddings import EmbeddingStorageStage
 from nv_ingest.framework.orchestration.ray.stages.transforms.image_caption import ImageCaptionTransformStage
 from nv_ingest.framework.orchestration.ray.stages.transforms.text_embed import TextEmbeddingTransformStage
-from nv_ingest.framework.orchestration.ray.stages.transforms.text_splitter import TextSplitterStage
 from nv_ingest.framework.schemas.framework_metadata_injector_schema import MetadataInjectorSchema
 from nv_ingest_api.internal.schemas.extract.extract_audio_schema import AudioExtractorSchema
 from nv_ingest_api.internal.schemas.extract.extract_chart_schema import ChartExtractorSchema

nv_ingest-25.6.28.dev20250628/nv_ingest/framework/orchestration/ray/util/pipeline/tools.py

@@ -0,0 +1,203 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import uuid
+from typing import Callable, Optional, Union, Dict, List, Type
+
+import ray
+from pydantic import BaseModel
+
+from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
+from nv_ingest.framework.util.flow_control import filter_by_task
+from nv_ingest_api.internal.primitives.tracing.tagging import traceable
+from nv_ingest_api.util.exception_handlers.decorators import nv_ingest_node_failure_try_except
+
+logger = logging.getLogger(__name__)
+
+
+def wrap_callable_as_stage(
+    fn: Callable[[object, BaseModel], object],
+    schema_type: Type[BaseModel],
+    *,
+    required_tasks: Optional[List[str]] = None,
+    trace_id: Optional[str] = None,
+):
+    """
+    Factory to wrap a user-supplied function into a Ray actor, returning a proxy
+    for unique, isolated dynamic actor creation.
+
+    Parameters
+    ----------
+    fn : Callable[[IngestControlMessage, BaseModel], IngestControlMessage]
+        The processing function to be wrapped in the Ray actor.
+    schema_type : Type[BaseModel]
+        Pydantic schema used to validate and pass the stage config.
+    required_tasks : Optional[List[str]], optional
+        Task names this stage should filter on. If None, no filtering is applied.
+    trace_id : Optional[str], optional
+        Optional name for trace annotation; defaults to the function name.
+
+    Returns
+    -------
+    StageProxy : object
+        A factory-like proxy exposing `.remote()` and `.options()` for Ray-idiomatic
+        actor creation. Direct instantiation or class method use is not supported.
+
+    Notes
+    -----
+    - Each call to `.remote()` or `.options()` generates a new, dynamically created class
+      (using `type()`), ensuring Ray treats each as a unique actor type and preventing
+      class/actor name collisions or registry issues. This is essential when running
+      dynamic or parallel pipelines and tests.
+    - Only `.remote(config)` and `.options(...)` (chained with `.remote(config)`) are supported.
+      All other class/actor patterns will raise `NotImplementedError`.
+    """
+    trace_name = trace_id or fn.__name__
+
+    def make_actor_class():
+        """
+        Dynamically constructs a unique Ray actor class for every call.
+
+        Engineering Note
+        ----------------
+        This pattern uses Python's `type()` to create a new class object for each actor instance,
+        guaranteeing a unique type each time. Ray's internal registry identifies actor types
+        by their Python class object. If you use the same class (even with different logic or
+        @ray.remote), Ray may reuse or overwrite them, causing hard-to-diagnose bugs in
+        parallel or test code. By generating a fresh class each time, we fully isolate state,
+        serialization, and Ray's registry—avoiding actor collisions and test pollution.
+
+        Returns
+        -------
+        new_class : type
+            The dynamically constructed RayActorStage subclass.
+        """
+        class_name = f"LambdaStage_{fn.__name__}_{uuid.uuid4().hex[:8]}"
+
+        def __init__(self, config: Union[Dict, BaseModel]) -> None:
+            """
+            Parameters
+            ----------
+            config : Union[Dict, BaseModel]
+                Stage configuration, validated against `schema_type`.
+            """
+            validated_config = schema_type(**config) if not isinstance(config, schema_type) else config
+            super(new_class, self).__init__(validated_config, log_to_stdout=True)
+            self.validated_config = validated_config
+            self._logger.info(f"{self.__class__.__name__} initialized with validated config.")
+
+        @traceable(trace_name)
+        @nv_ingest_node_failure_try_except(annotation_id=trace_name, raise_on_failure=False)
+        @filter_by_task(required_tasks=required_tasks) if required_tasks else (lambda f: f)
+        def on_data(self, control_message):
+            """
+            Processes a control message using the wrapped function.
+
+            Parameters
+            ----------
+            control_message : IngestControlMessage
+                The message to be processed.
+
+            Returns
+            -------
+            IngestControlMessage
+                The processed message, or the original on failure.
+            """
+            try:
+                return fn(control_message, self.validated_config)
+            except Exception as e:
+                self._logger.exception(f"{self.__class__.__name__} failed: {e}")
+                self.stats["errors"] += 1
+                return control_message
+
+        # --- ENGINEERING NOTE ---
+        # The `class_dict` collects all the methods and attributes for the dynamic class.
+        # This allows us to build a fresh class object per call, preventing Ray from
+        # reusing or overwriting global actor types. It is the critical piece for
+        # robust dynamic actor creation in Ray!
+        # ------------------------
+
+        class_dict = {
+            "__init__": __init__,
+            "on_data": on_data,
+        }
+        bases = (RayActorStage,)
+        new_class = type(class_name, bases, class_dict)
+        return new_class
+
+    class StageProxy:
+        """
+        Factory/proxy for dynamic Ray actor creation; not itself a Ray actor.
+
+        Methods
+        -------
+        remote(config)
+            Instantiate a Ray actor with a unique dynamic class and name.
+        options(*args, **kwargs)
+            Advanced Ray actor configuration (chain with `.remote(config)`).
+        actor_class()
+            Generates and returns a fresh actor class (for introspection/testing only).
+        """
+
+        @staticmethod
+        def remote(config):
+            """
+            Instantiate a Ray actor with a unique dynamic class and name.
+
+            Parameters
+            ----------
+            config : Union[Dict, BaseModel]
+                Stage configuration to pass to the actor.
+
+            Returns
+            -------
+            ray.actor.ActorHandle
+                Handle to the started Ray actor.
+            """
+            _ActorClass = ray.remote(make_actor_class())
+            unique_name = f"{fn.__name__}_{str(uuid.uuid4())[:8]}"
+            return _ActorClass.options(name=unique_name).remote(config)
+
+        @staticmethod
+        def options(*args, **kwargs):
+            """
+            Return a Ray actor class with the specified options set.
+            Must call `.remote(config)` on the result.
+
+            Parameters
+            ----------
+            *args
+                Positional arguments for Ray actor options.
+            **kwargs
+                Keyword arguments for Ray actor options (e.g., resources).
+
+            Returns
+            -------
+            ray.actor.ActorClass
+                Ray actor class, requires .remote(config) to instantiate.
+            """
+            ActorClass = ray.remote(make_actor_class())
+            if "name" not in kwargs:
+                kwargs["name"] = f"{fn.__name__}_{str(uuid.uuid4())[:8]}"
+            return ActorClass.options(*args, **kwargs)
+
+        def __new__(cls, *a, **k):
+            raise NotImplementedError("StageProxy is a factory, not a Ray actor or class. Use .remote() or .options().")
+
+        def __call__(self, *a, **k):
+            raise NotImplementedError("StageProxy is a factory, not a Ray actor or class. Use .remote() or .options().")
+
+        def __getattr__(self, name):
+            # Only allow access to known public members
+            if name in {"remote", "options", "actor_class"}:
+                return getattr(self, name)
+            raise NotImplementedError(
+                f"StageProxy does not implement '{name}'. Only .remote(), .options(), .actor_class() are available."
+            )
+
+        # For testing or introspection only.
+        # actor_class = staticmethod(make_actor_class)
+
+    return StageProxy

{nv_ingest-25.6.26.dev20250626 → nv_ingest-25.6.28.dev20250628}/nv_ingest.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest
-Version: 25.6.26.dev20250626
+Version: 25.6.28.dev20250628
 Summary: Python module for multimodal document ingestion
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest-25.6.26.dev20250626 → nv_ingest-25.6.28.dev20250628}/nv_ingest.egg-info/SOURCES.txt

@@ -76,6 +76,7 @@ nv_ingest/framework/orchestration/ray/util/pipeline/pid_controller.py
 nv_ingest/framework/orchestration/ray/util/pipeline/pipeline_builders.py
 nv_ingest/framework/orchestration/ray/util/pipeline/pipeline_runners.py
 nv_ingest/framework/orchestration/ray/util/pipeline/stage_builders.py
+nv_ingest/framework/orchestration/ray/util/pipeline/tools.py
 nv_ingest/framework/orchestration/ray/util/system_tools/__init__.py
 nv_ingest/framework/orchestration/ray/util/system_tools/memory.py
 nv_ingest/framework/orchestration/ray/util/system_tools/visualizers.py