nv-ingest 25.7.7.dev20250707__py3-none-any.whl → 25.8.0rc2__py3-none-any.whl

nv_ingest/framework/orchestration/ray/util/env_config.py

@@ -0,0 +1,75 @@
+import os
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def str_to_bool(value: str) -> bool:
+    """
+    Convert string to boolean value.
+
+    Parameters
+    ----------
+    value : str
+        String value to convert
+
+    Returns
+    -------
+    bool
+        Boolean representation of the string
+    """
+    return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def get_env_var(name: str, default, var_type=None):
+    """
+    Get environment variable with type conversion and default value.
+
+    Parameters
+    ----------
+    name : str
+        Environment variable name
+    default : Any
+        Default value if environment variable is not set
+    var_type : type, optional
+        Type to convert to. If None, infers from default value type
+
+    Returns
+    -------
+    Any
+        Environment variable value converted to the appropriate type
+    """
+    value = os.environ.get(name)
+    if value is None:
+        return default
+
+    # Determine type from default if not explicitly provided
+    target_type = var_type or type(default)
+
+    # Handle boolean conversion specially
+    if target_type is bool:
+        return str_to_bool(value)
+
+    # For other types, use direct conversion
+    try:
+        return target_type(value)
+    except (ValueError, TypeError) as e:
+        logger.warning(
+            f"Failed to convert environment variable {name}='{value}' to \
+                  {target_type.__name__}. Using default: {default}, error: {e}"
+        )
+        return default
+
+
+# Dynamic Memory Scaling Configuration
+DISABLE_DYNAMIC_SCALING = get_env_var("INGEST_DISABLE_DYNAMIC_SCALING", False, bool)
+DYNAMIC_MEMORY_THRESHOLD = get_env_var("INGEST_DYNAMIC_MEMORY_THRESHOLD", 0.75, float)
+DYNAMIC_MEMORY_KP = get_env_var("INGEST_DYNAMIC_MEMORY_KP", 0.2, float)
+DYNAMIC_MEMORY_KI = get_env_var("INGEST_DYNAMIC_MEMORY_KI", 0.01, float)
+DYNAMIC_MEMORY_EMA_ALPHA = get_env_var("INGEST_DYNAMIC_MEMORY_EMA_ALPHA", 0.1, float)
+DYNAMIC_MEMORY_TARGET_QUEUE_DEPTH = get_env_var("INGEST_DYNAMIC_MEMORY_TARGET_QUEUE_DEPTH", 0, int)
+DYNAMIC_MEMORY_PENALTY_FACTOR = get_env_var("INGEST_DYNAMIC_MEMORY_PENALTY_FACTOR", 0.1, float)
+DYNAMIC_MEMORY_ERROR_BOOST_FACTOR = get_env_var("INGEST_DYNAMIC_MEMORY_ERROR_BOOST_FACTOR", 1.5, float)
+DYNAMIC_MEMORY_RCM_MEMORY_SAFETY_BUFFER_FRACTION = get_env_var(
+    "INGEST_DYNAMIC_MEMORY_RCM_MEMORY_SAFETY_BUFFER_FRACTION", 0.15, float
+)

nv_ingest/framework/orchestration/ray/util/pipeline/pid_controller.py

@@ -6,9 +6,7 @@ import logging
 import math
 from dataclasses import dataclass
 
-import numpy as np
-from collections import deque
-from typing import Dict, Any, Deque, List, Tuple, Optional
+from typing import Dict, Any, List, Tuple, Optional
 
 from nv_ingest_api.util.system.hardware_info import SystemResourceProbe
 
@@ -16,7 +14,7 @@ logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 # --- Constants ---
-DEFAULT_STAGE_COST_MB = 5000.0  # Fallback memory cost
+DEFAULT_STAGE_COST_MB = 5_000.0  # Fallback memory cost
 
 
 @dataclass
@@ -46,9 +44,7 @@ class PIDController:
         kp: float,
         ki: float,
         kd: float,  # Currently unused in delta calculation
-        stage_cost_estimates: Dict[str, int],  # Static estimates (MB)
         target_queue_depth: int = 0,
-        window_size: int = 10,
         penalty_factor: float = 0.0005,
         error_boost_factor: float = 1.5,
     ):
@@ -64,16 +60,10 @@ class PIDController:
         kd : float
             Derivative gain. Reacts to the rate of change of the error.
             (Currently set to 0 in internal calculations).
-        stage_cost_estimates : Dict[str, int]
-            Static estimated memory cost (in MB) per replica for each stage.
-            Used as a fallback and minimum for dynamic estimates.
         target_queue_depth : int, optional
             Default target queue depth for stages if not specified in metrics,
             by default 0. The PID loop tries to drive the queue depth towards
             this value.
-        window_size : int, optional
-            Number of recent samples used for dynamic memory cost estimation
-            per replica, by default 10.
         penalty_factor : float, optional
             Multiplier applied to the number of consecutive idle cycles for a
             stage. The resulting penalty effectively lowers the target queue
@@ -90,16 +80,11 @@ class PIDController:
         self.error_boost_factor = error_boost_factor
 
         # Per-Stage State
-        self.stage_cost_estimates = {
-            name: float(max(cost, 1.0)) for name, cost in stage_cost_estimates.items()  # Ensure float and min 1MB
-        }
         self.integral_error: Dict[str, float] = {}
         self.prev_error: Dict[str, float] = {}
-        self.memory_history: Dict[str, Deque[float]] = {}  # Per-replica memory history (MB)
         self.idle_cycles: Dict[str, int] = {}
 
         # Per-Stage Config
-        self.window_size = window_size
         self.penalty_factor = penalty_factor
 
     # --- Private Methods ---
@@ -110,48 +95,7 @@ class PIDController:
             logger.debug(f"[PID-{stage}] Initializing state.")
             self.integral_error[stage] = 0.0
             self.prev_error[stage] = 0.0
-            self.memory_history[stage] = deque(maxlen=self.window_size)
             self.idle_cycles[stage] = 0
-            # Ensure static cost estimate exists, provide default if missing
-            if stage not in self.stage_cost_estimates:
-                logger.warning(f"[PID-{stage}] Missing static cost estimate. Using default {DEFAULT_STAGE_COST_MB}MB.")
-                self.stage_cost_estimates[stage] = DEFAULT_STAGE_COST_MB
-
-    def _get_conservative_cost_estimate(self, stage: str) -> float:
-        """
-        Estimates dynamic memory cost, using static estimate as a floor/max.
-
-        Returns the maximum of the recent average dynamic cost per replica
-        and the static estimate provided during initialization. This provides
-        a conservative value for resource projection.
-
-        Parameters
-        ----------
-        stage : str
-            The name of the stage.
-
-        Returns
-        -------
-        float
-            The conservative memory cost estimate in MB per replica.
-        """
-        static_cost = self.stage_cost_estimates.get(stage, DEFAULT_STAGE_COST_MB)
-        memory_samples = self.memory_history.get(stage)
-
-        # Use numpy.mean if samples exist, otherwise fallback to static
-        if memory_samples and len(memory_samples) > 0:
-            try:
-                dynamic_avg = float(np.mean(memory_samples))
-                # Use max(dynamic, static) for projection, enforce min 1MB
-                cost = max(dynamic_avg, static_cost, 1.0)
-                return cost
-            except Exception as e:
-                logger.error(
-                    f"[PID-{stage}] Error calculating mean of memory samples: {e}. Falling back to static cost.",
-                    exc_info=False,
-                )
-                return max(static_cost, 1.0)  # Fallback safely
-        return max(static_cost, 1.0)  # Fallback to static estimate if no history
 
     # --- Public Method ---
 
@@ -167,8 +111,8 @@ class PIDController:
         ----------
         stage_metrics : Dict[str, Dict[str, Any]]
             Dictionary mapping stage names to their current metrics. Expected keys
-            per stage: 'replicas', 'queue_depth'. Optional: 'memory_usage',
-            'target_queue_depth', 'processing', 'min_replicas', 'max_replicas'.
+            per stage: 'replicas', 'queue_depth', 'ema_memory_per_replica'.
+            Optional: 'target_queue_depth', 'processing', 'min_replicas', 'max_replicas'.
 
         Returns
         -------
@@ -185,16 +129,9 @@ class PIDController:
 
             # --- Extract data and calculate current memory state ---
             replicas = metrics.get("replicas", 0)
-            # Start with static cost as initial guess if no memory_usage provided
-            initial_cost_guess = self.stage_cost_estimates.get(stage, DEFAULT_STAGE_COST_MB)
-            memory_usage = metrics.get("memory_usage", initial_cost_guess * max(replicas, 1))
-            # Calculate memory per replica safely (avoid division by zero)
-            current_memory_per_replica = memory_usage / max(replicas, 1.0)
-
-            # Update memory history *before* calculating the conservative cost for *this* cycle's proposal
-            self.memory_history[stage].append(current_memory_per_replica)
-            # Recalculate conservative cost *after* updating history for the proposal
-            conservative_cost = self._get_conservative_cost_estimate(stage)
+            # The conservative cost is now the EMA memory passed in from the stats collector.
+            # Fallback to a default if not present.
+            conservative_cost = metrics.get("ema_memory_per_replica", DEFAULT_STAGE_COST_MB)
 
             # --- PID Calculation ---
             queue_depth = metrics.get("queue_depth", 0)
@@ -296,7 +233,6 @@ class ResourceConstraintManager:
         self,
         max_replicas: int,
         memory_threshold: int,
-        estimated_edge_cost_mb: int,
         memory_safety_buffer_fraction: float,
     ):
         """
@@ -309,7 +245,6 @@ class ResourceConstraintManager:
 
         self.max_replicas = max_replicas
         self.memory_threshold_mb = memory_threshold
-        self.estimated_edge_cost_mb = estimated_edge_cost_mb  # Keep track, though unused
         self.memory_safety_buffer_fraction = memory_safety_buffer_fraction  # Unused
         self.effective_memory_limit_mb = self.memory_threshold_mb
 

nv_ingest/framework/orchestration/ray/util/pipeline/pipeline_builders.py

@@ -9,6 +9,7 @@ import os
 from typing import Dict, Any
 
 import ray
+from ray import LoggingConfig
 from pydantic import BaseModel
 
 from nv_ingest.framework.orchestration.ray.primitives.ray_pipeline import RayPipeline
@@ -47,16 +48,174 @@ def export_config_to_env(ingest_config: Any) -> None:
     os.environ.update({key.upper(): val for key, val in ingest_config.items()})
 
 
+def build_logging_config_from_env() -> LoggingConfig:
+    """
+    Build Ray LoggingConfig from environment variables.
+
+    Package-level preset (sets all defaults):
+    - INGEST_RAY_LOG_LEVEL: PRODUCTION, DEVELOPMENT, DEBUG. Default: DEVELOPMENT
+
+    Individual environment variables (override preset defaults):
+    - RAY_LOGGING_LEVEL: Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL). Default: INFO
+    - RAY_LOGGING_ENCODING: Log encoding format (TEXT, JSON). Default: TEXT
+    - RAY_LOGGING_ADDITIONAL_ATTRS: Comma-separated list of additional standard logger attributes
+    - RAY_DEDUP_LOGS: Enable/disable log deduplication (0/1). Default: 1 (enabled)
+    - RAY_LOG_TO_DRIVER: Enable/disable logging to driver (true/false). Default: true
+    - RAY_LOGGING_ROTATE_BYTES: Maximum log file size before rotation (bytes). Default: 1GB
+    - RAY_LOGGING_ROTATE_BACKUP_COUNT: Number of backup log files to keep. Default: 19
+    - RAY_DISABLE_IMPORT_WARNING: Disable Ray import warnings (0/1). Default: 0
+    - RAY_USAGE_STATS_ENABLED: Enable/disable usage stats collection (0/1). Default: 1
+    """
+
+    # Apply package-level preset defaults first
+    preset_level = os.environ.get("INGEST_RAY_LOG_LEVEL", "DEVELOPMENT").upper()
+
+    # Define preset configurations
+    presets = {
+        "PRODUCTION": {
+            "RAY_LOGGING_LEVEL": "ERROR",
+            "RAY_LOGGING_ENCODING": "TEXT",
+            "RAY_LOGGING_ADDITIONAL_ATTRS": "",
+            "RAY_DEDUP_LOGS": "1",
+            "RAY_LOG_TO_DRIVER": "0",  # false
+            "RAY_LOGGING_ROTATE_BYTES": "1073741824",  # 1GB
+            "RAY_LOGGING_ROTATE_BACKUP_COUNT": "9",  # 10GB total
+            "RAY_DISABLE_IMPORT_WARNING": "1",
+            "RAY_USAGE_STATS_ENABLED": "0",
+        },
+        "DEVELOPMENT": {
+            "RAY_LOGGING_LEVEL": "INFO",
+            "RAY_LOGGING_ENCODING": "TEXT",
+            "RAY_LOGGING_ADDITIONAL_ATTRS": "",
+            "RAY_DEDUP_LOGS": "1",
+            "RAY_LOG_TO_DRIVER": "1",  # true
+            "RAY_LOGGING_ROTATE_BYTES": "1073741824",  # 1GB
+            "RAY_LOGGING_ROTATE_BACKUP_COUNT": "19",  # 20GB total
+            "RAY_DISABLE_IMPORT_WARNING": "0",
+            "RAY_USAGE_STATS_ENABLED": "1",
+        },
+        "DEBUG": {
+            "RAY_LOGGING_LEVEL": "DEBUG",
+            "RAY_LOGGING_ENCODING": "JSON",
+            "RAY_LOGGING_ADDITIONAL_ATTRS": "name,funcName,lineno",
+            "RAY_DEDUP_LOGS": "0",
+            "RAY_LOG_TO_DRIVER": "1",  # true
+            "RAY_LOGGING_ROTATE_BYTES": "536870912",  # 512MB
+            "RAY_LOGGING_ROTATE_BACKUP_COUNT": "39",  # 20GB total
+            "RAY_DISABLE_IMPORT_WARNING": "0",
+            "RAY_USAGE_STATS_ENABLED": "1",
+        },
+    }
+
+    # Validate preset level
+    if preset_level not in presets:
+        logger.warning(
+            f"Invalid INGEST_RAY_LOG_LEVEL '{preset_level}', using DEVELOPMENT. "
+            f"Valid presets: {list(presets.keys())}"
+        )
+        preset_level = "DEVELOPMENT"
+
+    # Apply preset defaults (only if env var not already set)
+    preset_config = presets[preset_level]
+    for key, default_value in preset_config.items():
+        if key not in os.environ:
+            os.environ[key] = default_value
+
+    logger.info(f"Applied Ray logging preset: {preset_level}")
+
+    # Get log level from environment, default to INFO
+    log_level = os.environ.get("RAY_LOGGING_LEVEL", "INFO").upper()
+
+    # Validate log level
+    valid_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
+    if log_level not in valid_levels:
+        logger.warning(f"Invalid RAY_LOGGING_LEVEL '{log_level}', using INFO. Valid levels: {valid_levels}")
+        log_level = "INFO"
+
+    # Get encoding format from environment, default to TEXT
+    encoding = os.environ.get("RAY_LOGGING_ENCODING", "TEXT").upper()
+
+    # Validate encoding
+    valid_encodings = ["TEXT", "JSON"]
+    if encoding not in valid_encodings:
+        logger.warning(f"Invalid RAY_LOGGING_ENCODING '{encoding}', using TEXT. Valid encodings: {valid_encodings}")
+        encoding = "TEXT"
+
+    # Get additional standard logger attributes
+    additional_attrs_str = os.environ.get("RAY_LOGGING_ADDITIONAL_ATTRS", "")
+    additional_log_standard_attrs = []
+    if additional_attrs_str:
+        additional_log_standard_attrs = [attr.strip() for attr in additional_attrs_str.split(",") if attr.strip()]
+
+    # Set log deduplication environment variable if specified
+    dedup_logs = os.environ.get("RAY_DEDUP_LOGS", "1")
+    if dedup_logs is not None:
+        os.environ["RAY_DEDUP_LOGS"] = str(dedup_logs)
+
+    # Set log to driver environment variable if specified
+    log_to_driver = os.environ.get("RAY_LOG_TO_DRIVER", "1")
+    if log_to_driver is not None:
+        os.environ["RAY_LOG_TO_DRIVER"] = str(log_to_driver).lower()
+
+    # Configure log rotation settings
+    rotate_bytes = os.environ.get("RAY_LOGGING_ROTATE_BYTES", "1073741824")  # Default: 1GB per file
+    if rotate_bytes is not None:
+        try:
+            rotate_bytes_int = int(rotate_bytes)
+            os.environ["RAY_LOGGING_ROTATE_BYTES"] = str(rotate_bytes_int)
+        except ValueError:
+            logger.warning(f"Invalid RAY_LOGGING_ROTATE_BYTES '{rotate_bytes}', using default (1GB)")
+            os.environ["RAY_LOGGING_ROTATE_BYTES"] = "1073741824"
+
+    rotate_backup_count = os.environ.get("RAY_LOGGING_ROTATE_BACKUP_COUNT", "19")  # Default: 19 backups (20GB Max)
+    if rotate_backup_count is not None:
+        try:
+            backup_count_int = int(rotate_backup_count)
+            os.environ["RAY_LOGGING_ROTATE_BACKUP_COUNT"] = str(backup_count_int)
+        except ValueError:
+            logger.warning(f"Invalid RAY_LOGGING_ROTATE_BACKUP_COUNT '{rotate_backup_count}', using default (19)")
+            os.environ["RAY_LOGGING_ROTATE_BACKUP_COUNT"] = "19"
+
+    # Configure Ray internal logging verbosity
+    disable_import_warning = os.environ.get("RAY_DISABLE_IMPORT_WARNING", "0")
+    if disable_import_warning is not None:
+        os.environ["RAY_DISABLE_IMPORT_WARNING"] = str(disable_import_warning)
+
+    # Configure usage stats collection
+    usage_stats_enabled = os.environ.get("RAY_USAGE_STATS_ENABLED", "1")
+    if usage_stats_enabled is not None:
+        os.environ["RAY_USAGE_STATS_ENABLED"] = str(usage_stats_enabled)
+
+    # Create LoggingConfig with validated parameters
+    logging_config = LoggingConfig(
+        encoding=encoding,
+        log_level=log_level,
+        additional_log_standard_attrs=additional_log_standard_attrs,
+    )
+
+    logger.info(
+        f"Ray logging configured: preset={preset_level}, level={log_level}, encoding={encoding}, "
+        f"additional_attrs={additional_log_standard_attrs}, "
+        f"dedup_logs={os.environ.get('RAY_DEDUP_LOGS', '1')}, "
+        f"log_to_driver={os.environ.get('RAY_LOG_TO_DRIVER', 'true')}, "
+        f"rotate_bytes={os.environ.get('RAY_LOGGING_ROTATE_BYTES', '1073741824')}, "
+        f"rotate_backup_count={os.environ.get('RAY_LOGGING_ROTATE_BACKUP_COUNT', '19')}"
+    )
+
+    return logging_config
+
+
 def setup_ingestion_pipeline(pipeline: RayPipeline, ingest_config: Dict[str, Any] = None):
     # Initialize the pipeline with the configuration
     if ingest_config:
         # Export the config to environment variables
         export_config_to_env(ingest_config)
 
-    current_level = logging.getLogger().getEffectiveLevel()
+    _ = logging.getLogger().getEffectiveLevel()
+    logging_config = build_logging_config_from_env()
     ray_context = ray.init(
         namespace="nv_ingest_ray",
-        logging_level=current_level,
+        logging_config=logging_config,
         ignore_reinit_error=True,
         dashboard_host="0.0.0.0",
         dashboard_port=8265,

nv_ingest/framework/orchestration/ray/util/pipeline/pipeline_runners.py

@@ -23,18 +23,21 @@ from nv_ingest.framework.orchestration.ray.primitives.ray_pipeline import (
     RayPipelineInterface,
 )
 from nv_ingest.framework.orchestration.ray.util.pipeline.pipeline_builders import setup_ingestion_pipeline
+from nv_ingest.framework.orchestration.ray.util.env_config import (
+    DISABLE_DYNAMIC_SCALING,
+    DYNAMIC_MEMORY_THRESHOLD,
+    DYNAMIC_MEMORY_KP,
+    DYNAMIC_MEMORY_KI,
+    DYNAMIC_MEMORY_EMA_ALPHA,
+    DYNAMIC_MEMORY_TARGET_QUEUE_DEPTH,
+    DYNAMIC_MEMORY_PENALTY_FACTOR,
+    DYNAMIC_MEMORY_ERROR_BOOST_FACTOR,
+    DYNAMIC_MEMORY_RCM_MEMORY_SAFETY_BUFFER_FRACTION,
+)
 
 logger = logging.getLogger(__name__)
 
 
-def str_to_bool(value: str) -> bool:
-    return value.strip().lower() in {"1", "true", "yes", "on"}
-
-
-DISABLE_DYNAMIC_SCALING = str_to_bool(os.environ.get("INGEST_DISABLE_DYNAMIC_SCALING", "false"))
-DYNAMIC_MEMORY_THRESHOLD = float(os.environ.get("INGEST_DYNAMIC_MEMORY_THRESHOLD", 0.75))
-
-
 class PipelineCreationSchema(BaseModel):
     """
     Schema for pipeline creation configuration.
@@ -78,15 +81,17 @@ class PipelineCreationSchema(BaseModel):
     otel_exporter_otlp_endpoint: str = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "localhost:4317")
 
     # OCR settings
-    paddle_http_endpoint: str = os.getenv("PADDLE_HTTP_ENDPOINT", "https://ai.api.nvidia.com/v1/cv/baidu/paddleocr")
-    paddle_infer_protocol: str = os.getenv("PADDLE_INFER_PROTOCOL", "http")
+    ocr_http_endpoint: str = os.getenv("OCR_HTTP_ENDPOINT", "https://ai.api.nvidia.com/v1/cv/baidu/paddleocr")
+    ocr_infer_protocol: str = os.getenv("OCR_INFER_PROTOCOL", "http")
+    ocr_model_name: str = os.getenv("OCR_MODEL_NAME", "paddle")
 
     # Task queue settings
     REDIS_INGEST_TASK_QUEUE: str = "ingest_task_queue"
 
     # Vision language model settings
     vlm_caption_endpoint: str = os.getenv(
-        "VLM_CAPTION_ENDPOINT", "https://ai.api.nvidia.com/v1/gr/nvidia/llama-3.1-nemotron-nano-vl-8b-v1/chat/completions"
+        "VLM_CAPTION_ENDPOINT",
+        "https://integrate.api.nvidia.com/v1/chat/completions",
     )
     vlm_caption_model_name: str = os.getenv("VLM_CAPTION_MODEL_NAME", "nvidia/llama-3.1-nemotron-nano-vl-8b-v1")
 
@@ -233,7 +238,15 @@ def _launch_pipeline(
     dynamic_memory_threshold = dynamic_memory_threshold if dynamic_memory_threshold else DYNAMIC_MEMORY_THRESHOLD
 
     scaling_config = ScalingConfig(
-        dynamic_memory_scaling=dynamic_memory_scaling, dynamic_memory_threshold=dynamic_memory_threshold
+        dynamic_memory_scaling=dynamic_memory_scaling,
+        dynamic_memory_threshold=dynamic_memory_threshold,
+        pid_kp=DYNAMIC_MEMORY_KP,
+        pid_ki=DYNAMIC_MEMORY_KI,
+        pid_ema_alpha=DYNAMIC_MEMORY_EMA_ALPHA,
+        pid_target_queue_depth=DYNAMIC_MEMORY_TARGET_QUEUE_DEPTH,
+        pid_penalty_factor=DYNAMIC_MEMORY_PENALTY_FACTOR,
+        pid_error_boost_factor=DYNAMIC_MEMORY_ERROR_BOOST_FACTOR,
+        rcm_memory_safety_buffer_fraction=DYNAMIC_MEMORY_RCM_MEMORY_SAFETY_BUFFER_FRACTION,
     )
 
     pipeline = RayPipeline(scaling_config=scaling_config)