nv-ingest 2025.8.4.dev20250804__py3-none-any.whl → 2025.12.10.dev20251210__py3-none-any.whl

nv_ingest/framework/orchestration/ray/stages/extractors/ocr_extractor.py

@@ -0,0 +1,71 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import ray
+
+from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
+from nv_ingest.framework.util.flow_control import filter_by_task
+from nv_ingest_api.internal.extract.image.ocr_extractor import extract_text_data_from_image_internal
+from nv_ingest_api.internal.primitives.ingest_control_message import IngestControlMessage, remove_task_by_type
+from nv_ingest_api.internal.primitives.tracing.tagging import traceable, set_trace_timestamps_with_parent_context
+from nv_ingest_api.internal.schemas.extract.extract_ocr_schema import OCRExtractorSchema
+from nv_ingest_api.util.exception_handlers.decorators import nv_ingest_node_failure_try_except
+from typing import Optional
+
+from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
+
+logger = logging.getLogger(__name__)
+
+
+@ray.remote
+class OCRExtractorStage(RayActorStage):
+    """
+    A Ray actor stage that extracts text data from image content.
+
+    It expects an IngestControlMessage containing a DataFrame with image data. It then:
+      1. Removes the "text_data_extract" task from the message.
+      2. Calls the text extraction logic using a validated configuration.
+      3. Updates the message payload with the extracted text DataFrame.
+    """
+
+    def __init__(self, config: OCRExtractorSchema, stage_name: Optional[str] = None) -> None:
+        super().__init__(config, log_to_stdout=False, stage_name=stage_name)
+        try:
+            self.validated_config = config
+            self._logger.info("OCRExtractorStage configuration validated successfully.")
+        except Exception as e:
+            self._logger.exception(f"Error validating Text extractor config: {e}")
+            raise
+
+    @nv_ingest_node_failure_try_except()
+    @traceable()
+    @udf_intercept_hook()
+    @filter_by_task(required_tasks=["ocr_data_extract"])
+    def on_data(self, control_message: IngestControlMessage) -> IngestControlMessage:
+        # Extract DataFrame payload
+        df_ledger = control_message.payload()
+        if df_ledger.empty:
+            return control_message
+
+        # Remove the "text_data_extract" task from the message
+        task_config = remove_task_by_type(control_message, "ocr_data_extract")
+
+        execution_trace_log = {}
+        new_df, extraction_info = extract_text_data_from_image_internal(
+            df_extraction_ledger=df_ledger,
+            task_config=task_config,
+            extraction_config=self.validated_config,
+            execution_trace_log=execution_trace_log,
+        )
+
+        control_message.payload(new_df)
+        control_message.set_metadata("ocr_extraction_info", extraction_info)
+
+        do_trace_tagging = control_message.get_metadata("config::add_trace_tagging") is True
+        if do_trace_tagging and execution_trace_log:
+            parent_name = self.stage_name if self.stage_name else "ocr_extractor"
+            set_trace_timestamps_with_parent_context(control_message, execution_trace_log, parent_name, logger)
+
+        return control_message

nv_ingest/framework/orchestration/ray/stages/extractors/pdf_extractor.py

@@ -7,16 +7,18 @@ import pandas as pd
 from typing import Any, Dict, Tuple, Optional
 import ray
 
-# Assume these imports come from your project:
-from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
-from nv_ingest.framework.util.flow_control import filter_by_task
 from nv_ingest_api.internal.extract.pdf.pdf_extractor import extract_primitives_from_pdf_internal
 from nv_ingest_api.internal.primitives.ingest_control_message import remove_task_by_type
-from nv_ingest_api.internal.primitives.tracing.tagging import traceable
 from nv_ingest_api.internal.schemas.extract.extract_pdf_schema import PDFExtractorSchema
+
+from nv_ingest_api.internal.primitives.tracing.tagging import set_trace_timestamps_with_parent_context, traceable
+from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
+from nv_ingest.framework.util.flow_control import filter_by_task
+from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
 from nv_ingest_api.util.exception_handlers.decorators import (
     nv_ingest_node_failure_try_except,
 )
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 logger = logging.getLogger(__name__)
 
@@ -51,19 +53,20 @@ class PDFExtractorStage(RayActorStage):
       4. Optionally, stores additional extraction info in the message metadata.
     """
 
-    def __init__(self, config: PDFExtractorSchema) -> None:
-        super().__init__(config)
+    def __init__(self, config: PDFExtractorSchema, stage_name: Optional[str] = None) -> None:
+        super().__init__(config, stage_name=stage_name)
         try:
             # Validate and store the PDF extractor configuration.
             self.validated_config = config
-            logger.info("PDFExtractorStage configuration validated successfully.")
+            logger.debug("PDFExtractorStage configuration validated successfully.")
         except Exception as e:
             logger.exception(f"Error validating PDF extractor config: {e}")
             raise
 
-    @traceable("pdf_extraction")
+    @nv_ingest_node_failure_try_except()
+    @traceable()
+    @udf_intercept_hook()
     @filter_by_task(required_tasks=[("extract", {"document_type": "pdf"})])
-    @nv_ingest_node_failure_try_except(annotation_id="pdf_extractor", raise_on_failure=False)
     def on_data(self, control_message: Any) -> Any:
         """
         Process the control message by extracting PDF content.
@@ -79,7 +82,7 @@ class PDFExtractorStage(RayActorStage):
             The updated message with the extracted DataFrame and extraction info in metadata.
         """
 
-        logger.info("PDFExtractorStage.on_data: Starting PDF extraction process.")
+        logger.debug("PDFExtractorStage.on_data: Starting PDF extraction process.")
 
         # Extract the DataFrame payload.
         df_extraction_ledger = control_message.payload()
@@ -87,7 +90,7 @@ class PDFExtractorStage(RayActorStage):
 
         # Remove the "extract" task from the message to obtain task-specific configuration.
         task_config = remove_task_by_type(control_message, "extract")
-        logger.debug("Extracted task config: %s", task_config)
+        logger.debug("Extracted task config: %s", sanitize_for_logging(task_config))
 
         # Perform PDF extraction.
         execution_trace_log = {}
@@ -97,17 +100,18 @@ class PDFExtractorStage(RayActorStage):
             execution_trace_log=execution_trace_log,
             validated_config=self.validated_config,
         )
-        logger.info("PDF extraction completed. Extracted %d rows.", len(new_df))
+        logger.debug("PDF extraction completed. Extracted %d rows.", len(new_df))
 
         # Update the message payload with the extracted DataFrame.
         control_message.payload(new_df)
         # Optionally, annotate the message with extraction info.
         control_message.set_metadata("pdf_extraction_info", extraction_info)
-        logger.info("PDF extraction metadata injected successfully.")
+        logger.debug("PDF extraction metadata injected successfully.")
 
         do_trace_tagging = control_message.get_metadata("config::add_trace_tagging") is True
         if do_trace_tagging and execution_trace_log:
-            for key, ts in execution_trace_log.items():
-                control_message.set_timestamp(key, ts)
+            # Use utility function to set trace timestamps with proper parent-child context
+            parent_name = self.stage_name or "pdf_extractor"
+            set_trace_timestamps_with_parent_context(control_message, execution_trace_log, parent_name, logger)
 
         return control_message

nv_ingest/framework/orchestration/ray/stages/extractors/pptx_extractor.py

@@ -3,8 +3,10 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import logging
+from typing import Optional
 
 import ray
+from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
 
 from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
 from nv_ingest.framework.util.flow_control import filter_by_task
@@ -28,7 +30,7 @@ class PPTXExtractorStage(RayActorStage):
       3. Updates the message payload with the extracted content DataFrame.
     """
 
-    def __init__(self, config: PPTXExtractorSchema) -> None:
+    def __init__(self, config: PPTXExtractorSchema, stage_name: Optional[str] = None) -> None:
         """
         Initializes the PptxExtractorStage.
 
@@ -36,8 +38,10 @@ class PPTXExtractorStage(RayActorStage):
         ----------
         config : PPTXExtractorSchema
             The validated configuration object for PPTX extraction.
+        stage_name : Optional[str]
+            Name of the stage from YAML pipeline configuration.
         """
-        super().__init__(config)
+        super().__init__(config, stage_name=stage_name)
         try:
             # The config passed in should already be validated, but storing it.
             self.validated_config = config
@@ -47,9 +51,10 @@ class PPTXExtractorStage(RayActorStage):
             logger.exception(f"Error initializing or validating PPTX Extractor config: {e}")
             raise
 
-    @traceable("pptx_extractor")
+    @nv_ingest_node_failure_try_except()
+    @traceable()
+    @udf_intercept_hook()
     @filter_by_task(required_tasks=[("extract", {"document_type": "pptx"})])
-    @nv_ingest_node_failure_try_except(annotation_id="pptx_extractor", raise_on_failure=False)
     def on_data(self, control_message: IngestControlMessage) -> IngestControlMessage:
         """
         Process the control message by extracting content from PPTX documents.
@@ -80,6 +85,6 @@ class PPTXExtractorStage(RayActorStage):
 
         # Update the message payload with the extracted PPTX content DataFrame.
         control_message.payload(new_df)
-        control_message.set_metadata("pptx_extraction_info", extraction_info)  # <-- Changed metadata key
+        control_message.set_metadata("pptx_extraction_info", extraction_info)
 
         return control_message

nv_ingest/framework/orchestration/ray/stages/extractors/table_extractor.py

@@ -3,19 +3,20 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import logging
-from typing import Any
+from typing import Any, Optional
 import ray
 
-# These imports are assumed from your project.
 from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
 from nv_ingest.framework.util.flow_control import filter_by_task
+from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
 from nv_ingest_api.internal.extract.image.table_extractor import extract_table_data_from_image_internal
 from nv_ingest_api.internal.primitives.ingest_control_message import remove_task_by_type
-from nv_ingest_api.internal.primitives.tracing.tagging import traceable
+from nv_ingest_api.internal.primitives.tracing.tagging import traceable, set_trace_timestamps_with_parent_context
 from nv_ingest_api.internal.schemas.extract.extract_table_schema import TableExtractorSchema
 from nv_ingest_api.util.exception_handlers.decorators import (
     nv_ingest_node_failure_try_except,
 )
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 logger = logging.getLogger(__name__)
 
@@ -31,18 +32,19 @@ class TableExtractorStage(RayActorStage):
     and annotates the message metadata with extraction info.
     """
 
-    def __init__(self, config: TableExtractorSchema) -> None:
-        super().__init__(config)
+    def __init__(self, config: TableExtractorSchema, stage_name: Optional[str] = None) -> None:
+        super().__init__(config, stage_name=stage_name)
         try:
             self.validated_config = config
-            logger.info("TableExtractorStage configuration validated successfully.")
+            logger.debug("TableExtractorStage configuration validated successfully.")
         except Exception as e:
             logger.exception("Error validating table extractor config")
             raise e
 
-    @traceable("table_extraction")
+    @nv_ingest_node_failure_try_except()
+    @traceable()
+    @udf_intercept_hook()
     @filter_by_task(required_tasks=["table_data_extract"])
-    @nv_ingest_node_failure_try_except(annotation_id="table_extraction", raise_on_failure=False)
     def on_data(self, control_message: Any) -> Any:
         """
         Process the control message by extracting table data from the PDF payload.
@@ -57,14 +59,14 @@ class TableExtractorStage(RayActorStage):
         IngestControlMessage
             The updated message with the extracted table data and extraction info in metadata.
         """
-        logger.info("TableExtractorStage.on_data: Starting table extraction.")
+        logger.debug("TableExtractorStage.on_data: Starting table extraction.")
         # Extract the DataFrame payload.
         df_payload = control_message.payload()
         logger.debug("Extracted payload with %d rows.", len(df_payload))
 
         # Remove the "table_data_extract" task to obtain task-specific configuration.
         task_config = remove_task_by_type(control_message, "table_data_extract")
-        logger.debug("Extracted task configuration: %s", task_config)
+        logger.debug("Extracted task configuration: %s", sanitize_for_logging(task_config))
 
         # Perform table data extraction.
         execution_trace_log = {}
@@ -74,17 +76,17 @@ class TableExtractorStage(RayActorStage):
             extraction_config=self.validated_config,
             execution_trace_log=execution_trace_log,
         )
-        logger.info("Table extraction completed. Extracted %d rows.", len(new_df))
+        logger.debug("Table extraction completed. Extracted %d rows.", len(new_df))
 
         # Update the control message with the new DataFrame.
         control_message.payload(new_df)
         # Annotate the message with extraction info.
         control_message.set_metadata("table_extraction_info", extraction_info)
-        logger.info("Table extraction metadata injected successfully.")
+        logger.debug("Table extraction metadata injected successfully.")
 
         do_trace_tagging = control_message.get_metadata("config::add_trace_tagging") is True
         if do_trace_tagging and execution_trace_log:
-            for key, ts in execution_trace_log.items():
-                control_message.set_timestamp(key, ts)
+            parent_name = self.stage_name if self.stage_name else "table_extractor"
+            set_trace_timestamps_with_parent_context(control_message, execution_trace_log, parent_name, logger)
 
         return control_message

nv_ingest/framework/orchestration/ray/stages/injectors/metadata_injector.py

@@ -4,12 +4,14 @@
 
 from datetime import datetime
 import logging
+from typing import Optional
 import pandas as pd
-from typing import Any
 from pydantic import BaseModel
 import ray
 
 from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
+from nv_ingest_api.internal.primitives.ingest_control_message import IngestControlMessage
+from nv_ingest_api.internal.primitives.tracing.tagging import traceable
 from nv_ingest_api.internal.enums.common import (
     DocumentTypeEnum,
     ContentTypeEnum,
@@ -17,14 +19,14 @@ from nv_ingest_api.internal.enums.common import (
     TextTypeEnum,
     LanguageEnum,
 )
-from nv_ingest_api.internal.primitives.tracing.tagging import traceable
 from nv_ingest_api.internal.schemas.meta.metadata_schema import ContentHierarchySchema
 from nv_ingest_api.util.converters.type_mappings import doc_type_to_content_type
 from nv_ingest_api.util.exception_handlers.decorators import (
     nv_ingest_node_failure_try_except,
 )
+from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
-# logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
 
 
@@ -37,15 +39,16 @@ class MetadataInjectionStage(RayActorStage):
     injection is required, and if so, injects the appropriate metadata.
     """
 
-    def __init__(self, config: BaseModel) -> None:
+    def __init__(self, config: BaseModel, stage_name: Optional[str] = None) -> None:
         # Call the base initializer to set attributes like self._running.
-        super().__init__(config)
+        super().__init__(config, stage_name=stage_name)
         # Additional initialization can be added here if necessary.
-        logger.info("MetadataInjectionStage initialized with config: %s", config)
+        self._logger.debug("MetadataInjectionStage initialized with config: %s", sanitize_for_logging(config))
 
-    @traceable("metadata_injector")
-    @nv_ingest_node_failure_try_except(annotation_id="metadata_injector", raise_on_failure=False)
-    def on_data(self, message: Any) -> Any:
+    @nv_ingest_node_failure_try_except()
+    @traceable()
+    @udf_intercept_hook()
+    def on_data(self, message: IngestControlMessage) -> IngestControlMessage:
         """
         Process an incoming IngestControlMessage by injecting metadata into its DataFrame payload.
 
@@ -62,7 +65,7 @@ class MetadataInjectionStage(RayActorStage):
         df = message.payload()
         update_required = False
         rows = []
-        logger.info("Starting metadata injection on DataFrame with %d rows", len(df))
+        logger.debug("Starting metadata injection on DataFrame with %d rows", len(df))
 
         for _, row in df.iterrows():
             try:
@@ -141,7 +144,7 @@ class MetadataInjectionStage(RayActorStage):
                         "source_metadata": default_source_metadata,
                         "text_metadata": default_text_metadata,
                     }
-                    logger.info(
+                    logger.debug(
                         f"METADATA_INJECTOR_DEBUG: Rebuilt metadata for source_id='{row.get('source_id', 'N/A')}'. "
                         f"Metadata keys: {list(row['metadata'].keys())}."
                         f"'content' present: {'content' in row['metadata']}"
@@ -154,8 +157,8 @@ class MetadataInjectionStage(RayActorStage):
         if update_required:
             docs = pd.DataFrame(rows)
             message.payload(docs)
-            logger.info("Metadata injection updated payload with %d rows", len(docs))
+            logger.debug("Metadata injection updated payload with %d rows", len(docs))
         else:
-            logger.info("No metadata update was necessary during metadata injection")
+            logger.debug("No metadata update was necessary during metadata injection")
 
         return message

nv_ingest/framework/orchestration/ray/stages/meta/ray_actor_sink_stage_base.py

@@ -21,6 +21,9 @@ class RayActorSinkStage(RayActorStage, ABC):
     to deliver their final processed messages.
     """
 
+    def __init__(self, config: Any, log_to_stdout=False, stage_name: Optional[str] = None) -> None:
+        super().__init__(config, log_to_stdout=log_to_stdout, stage_name=stage_name)
+
     @ray.method(num_returns=1)
     def set_output_queue(self, queue_handle: any) -> bool:
         raise NotImplementedError("Sink stages do not support an output queue.")

nv_ingest/framework/orchestration/ray/stages/meta/ray_actor_source_stage_base.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from abc import ABC, abstractmethod
-from typing import Any
+from typing import Any, Optional
 import ray
 import logging
 
@@ -19,8 +19,8 @@ class RayActorSourceStage(RayActorStage, ABC):
     Instead, they must implement get_input() to fetch control messages from an external source.
     """
 
-    def __init__(self, config: Any, log_to_stdout=False) -> None:
-        super().__init__(config, log_to_stdout=log_to_stdout)
+    def __init__(self, config: Any, log_to_stdout=False, stage_name: Optional[str] = None) -> None:
+        super().__init__(config, log_to_stdout=log_to_stdout, stage_name=stage_name)
         self.paused = False
 
     def on_data(self, IngestControlMessage):

nv_ingest/framework/orchestration/ray/stages/meta/ray_actor_stage_base.py

@@ -2,6 +2,7 @@
 # All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+import gc
 import sys
 import threading
 import time
@@ -14,6 +15,7 @@ import ray
 import ray.actor
 from pydantic import BaseModel
 import logging
+import pyarrow as pa
 
 from ray import get_runtime_context
 
@@ -49,6 +51,9 @@ class RayActorStage(ABC):
     ----------
     config : BaseModel
         Configuration object for the stage.
+    stage_name : Optional[str]
+        Name of the stage from YAML pipeline configuration. Used by
+        stage-aware decorators for consistent naming.
     _input_queue : Optional[Any]
         Handle to the Ray queue from which input items are read.
         Expected to be set via `set_input_queue`.
@@ -80,7 +85,7 @@ class RayActorStage(ABC):
         Lock to protect access to shutdown-related state (`_shutting_down`).
     """
 
-    def __init__(self, config: BaseModel, log_to_stdout=False) -> None:
+    def __init__(self, config: BaseModel, stage_name: Optional[str] = None, log_to_stdout=False) -> None:
         """
         Initialize the RayActorStage.
 
@@ -89,8 +94,14 @@ class RayActorStage(ABC):
         config : BaseModel
             Configuration object specific to the stage's behavior. Passed by
             the orchestrator during actor creation.
+        stage_name : Optional[str]
+            Name of the stage from YAML pipeline configuration. Used by
+            stage-aware decorators for consistent naming.
+        log_to_stdout : bool
+            Whether to enable stdout logging.
         """
         self.config: BaseModel = config
+        self.stage_name: Optional[str] = stage_name
         self._input_queue: Optional[Any] = None  # Ray Queue handle expected
         self._output_queue: Optional[Any] = None  # Ray Queue handle expected
         self._running: bool = False
@@ -129,6 +140,14 @@ class RayActorStage(ABC):
 
         self._actor_id_str = self._get_actor_id_str()
 
+        # --- PyArrow Memory Management ---
+        # Time-based periodic cleanup to prevent long-term memory accumulation
+        self._memory_cleanup_interval_seconds = getattr(
+            config, "memory_cleanup_interval_seconds", 300
+        )  # 5 minutes default
+        self._last_memory_cleanup_time = time.time()
+        self._memory_cleanups_performed = 0
+
     @staticmethod
     def _get_actor_id_str() -> str:
         """
@@ -344,6 +363,16 @@ class RayActorStage(ABC):
                     # This is the primary path for "successful processing".
                     self.stats["processed"] += 1
 
+                    # Time-based PyArrow memory cleanup check (best-effort, low overhead)
+                    try:
+                        current_time = time.time()
+                        if (current_time - self._last_memory_cleanup_time) >= self._memory_cleanup_interval_seconds:
+                            self._force_arrow_memory_cleanup()
+                            self._last_memory_cleanup_time = current_time
+                    except Exception:
+                        # Never allow cleanup issues to interfere with processing
+                        pass
+
                 except ray.exceptions.ObjectLostError:
                     # This error is handled inside the loop to prevent the actor from crashing.
                     # We log it and continue to the next message.
@@ -386,10 +415,69 @@ class RayActorStage(ABC):
             # This block executes when the processing thread is about to exit,
             # either due to self._running becoming False or an unhandled critical exception.
             self._logger.debug(f"[{self._actor_id_str}] Processing loop thread finished.")
+            # Perform a best-effort final memory cleanup on exit
+            try:
+                self._force_arrow_memory_cleanup()
+            except Exception:
+                pass
             # Signal that this actor's processing duties are complete.
             # External monitors (e.g., via a future from stop()) can use this signal.
             self._shutdown_signal_complete = True
 
+    def _force_arrow_memory_cleanup(self) -> None:
+        """
+        Best-effort memory cleanup for PyArrow allocations.
+
+        - Runs Python garbage collection to drop unreachable references.
+        - If PyArrow is available and its default memory pool supports
+          release_unused(), request it to return free pages to the OS.
+
+        Designed to be safe to call periodically; any failures are logged at
+        debug/warning levels and are non-fatal.
+        """
+        try:
+            # First, trigger Python GC to maximize reclaimable memory
+            gc.collect()
+
+            try:
+                pool = pa.default_memory_pool()
+                try:
+                    before_bytes = getattr(pool, "bytes_allocated", lambda: 0)()
+                except Exception:
+                    before_bytes = 0
+
+                released = False
+                if hasattr(pool, "release_unused"):
+                    try:
+                        pool.release_unused()
+                        released = True
+                    except Exception as e_release:
+                        self._logger.debug(f"[{self._actor_id_str}] Arrow pool release_unused() failed: {e_release}")
+
+                try:
+                    after_bytes = getattr(pool, "bytes_allocated", lambda: before_bytes)()
+                except Exception:
+                    after_bytes = before_bytes
+
+                if released:
+                    delta_mb = max(0, (before_bytes - after_bytes) / (1024 * 1024))
+                    if delta_mb > 0:
+                        self._logger.debug(
+                            f"[{self._actor_id_str}] Arrow cleanup released ~{delta_mb:.2f}"
+                            f" MB (pool now {after_bytes/(1024*1024):.2f} MB)."
+                        )
+                self._memory_cleanups_performed += 1
+            except ModuleNotFoundError:
+                # PyArrow not present; nothing to do beyond GC.
+                self._memory_cleanups_performed += 1
+            except Exception as e_pa:
+                # Any other PyArrow-related issues are non-fatal.
+                self._logger.debug(f"[{self._actor_id_str}] Arrow cleanup skipped due to error: {e_pa}")
+                self._memory_cleanups_performed += 1
+        except Exception as e:
+            # As a last resort, swallow any errors to avoid interfering with the actor loop.
+            self._logger.debug(f"[{self._actor_id_str}] Memory cleanup encountered an error: {e}")
+
     def _get_memory_usage_mb(self) -> float:
         """
         Gets the total memory usage of the current actor process (RSS).
@@ -500,7 +588,7 @@ class RayActorStage(ABC):
             self._logger.warning(f"{self._actor_id_str}: Start called but actor is already running.")
             return False
 
-        self._logger.info(f"{self._actor_id_str}: Starting actor...")
+        self._logger.debug(f"{self._actor_id_str}: Starting actor...")
         # --- Initialize Actor State ---
         self._running = True
         self._shutting_down = False  # Reset shutdown flag on start
@@ -519,14 +607,14 @@ class RayActorStage(ABC):
         )
         self._processing_thread.start()
 
-        self._logger.info(f"{self._actor_id_str}: Actor started successfully.")
+        self._logger.debug(f"{self._actor_id_str}: Actor started successfully.")
 
         return True
 
     @ray.method(num_returns=0)
     def stop(self) -> None:
         """Stops the actor's processing loop by setting the running flag to False."""
-        self._logger.info(f"[{self._actor_id_str}] Stop signal received. Initiating graceful shutdown.")
+        self._logger.debug(f"[{self._actor_id_str}] Stop signal received. Initiating graceful shutdown.")
         self._running = False
 
     def is_shutdown_complete(self) -> bool:

nv_ingest/framework/orchestration/ray/stages/mutate/image_dedup.py

@@ -4,11 +4,13 @@
 
 
 import logging
+from typing import Optional
 
 import ray
 
 from nv_ingest.framework.orchestration.ray.stages.meta.ray_actor_stage_base import RayActorStage
 from nv_ingest.framework.util.flow_control import filter_by_task
+from nv_ingest.framework.util.flow_control.udf_intercept import udf_intercept_hook
 from nv_ingest_api.internal.mutate.deduplicate import deduplicate_images_internal
 from nv_ingest_api.internal.primitives.ingest_control_message import IngestControlMessage, remove_task_by_type
 from nv_ingest_api.internal.primitives.tracing.tagging import traceable
@@ -16,6 +18,7 @@ from nv_ingest_api.internal.schemas.mutate.mutate_image_dedup_schema import Imag
 from nv_ingest_api.util.exception_handlers.decorators import (
     nv_ingest_node_failure_try_except,
 )
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 logger = logging.getLogger(__name__)
 
@@ -31,18 +34,19 @@ class ImageDedupStage(RayActorStage):
       3. Updates the message payload with the deduplicated DataFrame.
     """
 
-    def __init__(self, config: ImageDedupSchema) -> None:
-        super().__init__(config)
+    def __init__(self, config: ImageDedupSchema, stage_name: Optional[str] = None) -> None:
+        super().__init__(config, stage_name=stage_name)
         try:
             self.validated_config = config
-            logger.info("ImageDedupStage configuration validated successfully.")
+            logger.debug("ImageDedupStage configuration validated successfully.")
         except Exception as e:
             logger.exception(f"Error validating Image Deduplication config: {e}")
             raise
 
-    @traceable("image_deduplication")
+    @nv_ingest_node_failure_try_except()
+    @traceable()
+    @udf_intercept_hook()
     @filter_by_task(required_tasks=["dedup"])
-    @nv_ingest_node_failure_try_except(annotation_id="image_dedup", raise_on_failure=False)
     def on_data(self, control_message: IngestControlMessage) -> IngestControlMessage:
         """
         Process the control message by deduplicating images.
@@ -57,7 +61,7 @@ class ImageDedupStage(RayActorStage):
         IngestControlMessage
             The updated message with deduplicated images in the payload.
         """
-        logger.info("ImageDedupStage.on_data: Starting image deduplication process.")
+        logger.debug("ImageDedupStage.on_data: Starting image deduplication process.")
         try:
             # Extract the DataFrame payload.
             df_ledger = control_message.payload()
@@ -65,7 +69,7 @@ class ImageDedupStage(RayActorStage):
 
             # Remove the "dedup" task from the message to obtain task-specific configuration.
             task_config = remove_task_by_type(control_message, "dedup")
-            logger.debug("Extracted task config: %s", task_config)
+            logger.debug("Extracted task config: %s", sanitize_for_logging(task_config))
 
             # Perform image deduplication.
             new_df = deduplicate_images_internal(
@@ -74,7 +78,7 @@ class ImageDedupStage(RayActorStage):
                 mutate_config=self.validated_config,
                 execution_trace_log=None,
             )
-            logger.info("Image deduplication completed. Resulting DataFrame has %d rows.", len(new_df))
+            logger.debug("Image deduplication completed. Resulting DataFrame has %d rows.", len(new_df))
 
             # Update the message payload with the deduplicated DataFrame.
             control_message.payload(new_df)