nemo-evaluator 0.1.41__py3-none-any.whl → 0.1.71__py3-none-any.whl

nemo_evaluator/adapters/adapter_config.py

@@ -13,9 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 from typing import Any
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, ValidationError
+
+from nemo_evaluator.logging import get_logger
 
 
 class DiscoveryConfig(BaseModel):
@@ -58,9 +61,122 @@ class PostEvalHookConfig(BaseModel):
         use_enum_values = True
 
 
+class LegacyAdapterConfig(BaseModel):
+    """Legacy adapter configuration parameters (pre-interceptor format).
+
+    This model validates legacy configuration dictionaries to catch typos
+    and invalid parameters early, before conversion to the new interceptor format.
+    """
+
+    class Config:
+        extra = "forbid"  # Reject any extra fields not defined here
+
+    # Boolean flags for optional features
+    use_caching: bool = Field(default=True, description="Enable caching interceptor")
+    save_responses: bool = Field(default=False, description="Save responses to disk")
+    save_requests: bool = Field(default=False, description="Save requests to disk")
+    use_system_prompt: bool = Field(
+        default=False, description="Enable system prompt modification"
+    )
+    use_omni_info: bool = Field(
+        default=False, description="Enable omni info processing"
+    )
+    use_request_logging: bool = Field(
+        default=False, description="Enable request logging"
+    )
+    use_nvcf: bool = Field(default=False, description="Enable NVCF integration")
+    use_response_logging: bool = Field(
+        default=False, description="Enable response logging"
+    )
+    use_reasoning: bool = Field(
+        default=False, description="Enable reasoning token processing"
+    )
+    process_reasoning_traces: bool = Field(
+        default=False, description="Process reasoning traces"
+    )
+    use_progress_tracking: bool = Field(
+        default=False, description="Enable progress tracking"
+    )
+    use_raise_client_errors: bool = Field(
+        default=False, description="Raise client errors"
+    )
+    include_json: bool = Field(default=True, description="Include JSON in responses")
+
+    # Model fields that are also part of AdapterConfig
+    mode: str = Field(
+        default="server", description="Adapter mode: 'server' or 'client'"
+    )
+    generate_html_report: bool = Field(default=True, description="Generate HTML report")
+    html_report_size: int | None = Field(default=5, description="HTML report size")
+    tracking_requests_stats: bool = Field(
+        default=True, description="Track request statistics"
+    )
+    log_failed_requests: bool = Field(default=False, description="Log failed requests")
+    endpoint_type: str = Field(default="chat", description="Endpoint type")
+    caching_dir: str | None = Field(default=None, description="Caching directory")
+
+    # Optional string/dict configuration parameters
+    custom_system_prompt: str | None = Field(
+        default=None, description="Custom system prompt"
+    )
+    output_dir: str | None = Field(default=None, description="Output directory")
+    params_to_add: dict[str, Any] | None = Field(
+        default=None, description="Parameters to add"
+    )
+    params_to_remove: list[str] | None = Field(
+        default=None, description="Parameters to remove"
+    )
+    params_to_rename: dict[str, str] | None = Field(
+        default=None, description="Parameters to rename"
+    )
+
+    # Optional integer limits
+    max_logged_requests: int | None = Field(
+        default=None, description="Max logged requests"
+    )
+    max_logged_responses: int | None = Field(
+        default=None, description="Max logged responses"
+    )
+    max_saved_requests: int | None = Field(
+        default=None, description="Max saved requests"
+    )
+    max_saved_responses: int | None = Field(
+        default=None, description="Max saved responses"
+    )
+
+    # Reasoning-specific parameters
+    start_reasoning_token: str | None = Field(
+        default=None, description="Start reasoning token"
+    )
+    include_if_reasoning_not_finished: bool | None = Field(
+        default=None, description="Include unfinished reasoning"
+    )
+    track_reasoning: bool | None = Field(default=None, description="Track reasoning")
+    end_reasoning_token: str = Field(
+        default="</think>", description="End reasoning token"
+    )
+
+    # Progress tracking parameters
+    progress_tracking_url: str | None = Field(
+        default=None, description="Progress tracking URL"
+    )
+    progress_tracking_interval: int = Field(
+        default=1, description="Progress tracking interval"
+    )
+
+    # Logging parameters
+    logging_aggregated_stats_interval: int = Field(
+        default=100, description="Logging aggregated stats interval"
+    )
+
+
 class AdapterConfig(BaseModel):
     """Adapter configuration with registry-based interceptor support"""
 
+    mode: str = Field(
+        description="Adapter mode: 'server' (default) or 'client'",
+        default="server",
+    )
     discovery: DiscoveryConfig = Field(
         description="Configuration for discovering 3rd party modules and directories",
         default_factory=DiscoveryConfig,
@@ -82,48 +198,6 @@ class AdapterConfig(BaseModel):
         default=False,
     )
 
-    @classmethod
-    def get_legacy_defaults(cls) -> dict[str, Any]:
-        """Get default values for legacy configuration parameters."""
-        return {
-            "generate_html_report": True,
-            "html_report_size": 5,
-            "tracking_requests_stats": True,
-            "caching_dir": None,
-            "log_failed_requests": cls.model_fields["log_failed_requests"].default,
-            "endpoint_type": cls.model_fields["endpoint_type"].default,
-            # Boolean defaults for optional features
-            "use_caching": True,
-            "save_responses": False,
-            "save_requests": False,
-            "use_system_prompt": False,
-            "use_omni_info": False,
-            "use_request_logging": False,
-            "use_nvcf": False,
-            "use_response_logging": False,
-            "use_reasoning": False,
-            "process_reasoning_traces": False,
-            "use_progress_tracking": False,
-            "use_raise_client_errors": False,
-            "include_json": True,
-            "custom_system_prompt": None,
-            "output_dir": None,
-            "params_to_add": None,
-            "params_to_remove": None,
-            "params_to_rename": None,
-            "max_logged_requests": None,
-            "max_logged_responses": None,
-            "max_saved_requests": None,
-            "max_saved_responses": None,
-            "start_reasoning_token": None,
-            "include_if_reasoning_not_finished": None,
-            "track_reasoning": None,
-            "end_reasoning_token": "</think>",
-            "progress_tracking_url": None,
-            "progress_tracking_interval": 1,
-            "logging_aggregated_stats_interval": 100,
-        }
-
     @classmethod
     def get_validated_config(cls, run_config: dict[str, Any]) -> "AdapterConfig":
         """Extract and validate adapter configuration from run_config.
@@ -156,9 +230,9 @@ class AdapterConfig(BaseModel):
         )
 
         # Validate that legacy parameters are not mixed with interceptors
-        legacy_defaults = cls.get_legacy_defaults()
+        legacy_params = set(LegacyAdapterConfig.model_fields.keys())
         model_fields = set(cls.model_fields.keys())
-        legacy_only_params = set(legacy_defaults.keys()) - model_fields
+        legacy_only_params = legacy_params - model_fields
 
         for config_name, config in [
             ("global_adapter_config", global_cfg),
@@ -203,14 +277,19 @@ class AdapterConfig(BaseModel):
                 {"name": s} if isinstance(s, str) else s
                 for s in merged["post_eval_hooks"]
             ]
+
         try:
             config = cls(**merged)
 
             # If no interceptors are configured, try to convert from legacy format
             if not config.interceptors:
+                # Pass mode through merged config so it's preserved in legacy conversion
                 config = cls.from_legacy_config(merged, run_config)
 
             return config
+        except ValidationError:
+            # Re-raise ValidationError directly for clear error messages
+            raise
         except Exception as e:
             raise ValueError(f"Invalid adapter configuration: {e}") from e
 
@@ -274,10 +353,29 @@ class AdapterConfig(BaseModel):
 
         Returns:
             AdapterConfig instance with interceptors based on legacy config
+
+        Raises:
+            ValidationError: If legacy_config contains typos or invalid field names
         """
-        # Merge legacy config with defaults to avoid repeated .get() calls
-        defaults = cls.get_legacy_defaults()
-        legacy_config = {**defaults, **legacy_config}
+        logger = get_logger(__name__)
+
+        # Validate legacy config using Pydantic model (catches typos early)
+        # Filter out modern fields (discovery, interceptors, post_eval_hooks) before validation
+        modern_fields = {"discovery", "interceptors", "post_eval_hooks"}
+        legacy_only = {k: v for k, v in legacy_config.items() if k not in modern_fields}
+
+        try:
+            validated = LegacyAdapterConfig(**legacy_only)
+            legacy_config = validated.model_dump()
+        except ValidationError:
+            # Log helpful message with list of valid fields
+            valid_fields = sorted(LegacyAdapterConfig.model_fields.keys())
+            logger.error(
+                f"Invalid legacy adapter configuration. "
+                f"Supported parameters: {', '.join(valid_fields)}"
+            )
+            # Re-raise the original ValidationError
+            raise
 
         interceptors = []
         post_eval_hooks = []
@@ -474,8 +572,6 @@ class AdapterConfig(BaseModel):
             )
 
         if legacy_config["use_reasoning"]:
-            from nemo_evaluator.logging import get_logger
-
             logger = get_logger(__name__)
             logger.warning(
                 '"use_reasoning" is deprecated as it might suggest it touches on switching on/off reasoning for mode when it does not. Use "process_reasoning_traces" instead.'
@@ -543,7 +639,6 @@ class AdapterConfig(BaseModel):
             from nemo_evaluator.adapters.interceptors.raise_client_error_interceptor import (
                 RaiseClientErrorInterceptor,
             )
-            from nemo_evaluator.logging import get_logger
 
             logger = get_logger(__name__)
 
@@ -583,6 +678,7 @@ class AdapterConfig(BaseModel):
             )
 
         return cls(
+            mode=legacy_config["mode"],
             interceptors=interceptors,
             post_eval_hooks=post_eval_hooks,
             endpoint_type=legacy_config["endpoint_type"],

nemo_evaluator/adapters/interceptors/endpoint_interceptor.py

@@ -15,6 +15,7 @@
 
 """Endpoint interceptor that makes actual requests to the upstream API."""
 
+import json
 import time
 from typing import final
 
@@ -80,15 +81,43 @@ class EndpointInterceptor(RequestToResponseInterceptor):
         start_time = time.time()
 
         # This is a final interceptor, we'll need the flask_request and api
+        raw_response = requests.request(
+            method=ar.r.method,
+            url=context.url,
+            headers={k: v for k, v in ar.r.headers if k.lower() != "host"},
+            json=ar.r.json,
+            cookies=ar.r.cookies,
+            allow_redirects=False,
+        )
+
+        # replace choices[xx].message.content=None with empty string
+        if raw_response.content is not None:
+            try:
+                response_json = json.loads(raw_response.content)
+                if (
+                    "choices" in response_json
+                    and isinstance(response_json["choices"], list)
+                    and len(response_json["choices"]) > 0
+                ):
+                    for i, choice in enumerate(response_json["choices"]):
+                        if (
+                            "message" in choice
+                            and "content" in choice["message"]
+                            and choice["message"]["content"] is None
+                        ):
+                            self.logger.warning(
+                                f"choices[{i}].message.content is None, replacing with empty string"
+                            )
+                            choice["message"]["content"] = ""
+                raw_response._content = json.dumps(response_json).encode("utf-8")
+            except (json.JSONDecodeError, TypeError, KeyError) as e:
+                # If JSON parsing fails or unexpected structure, leave response unchanged
+                self.logger.debug(
+                    "Could not parse response as JSON, leaving unchanged", error=str(e)
+                )
+
         resp = AdapterResponse(
-            r=requests.request(
-                method=ar.r.method,
-                url=context.url,
-                headers={k: v for k, v in ar.r.headers if k.lower() != "host"},
-                json=ar.r.json,
-                cookies=ar.r.cookies,
-                allow_redirects=False,
-            ),
+            r=raw_response,
             rctx=ar.rctx,
             latency_ms=round(
                 (time.time() - start_time) * 1000, 2

nemo_evaluator/adapters/interceptors/reasoning_interceptor.py

@@ -127,6 +127,8 @@ class ResponseReasoningInterceptor(ResponseInterceptor, PostEvalHook):
             "responses_with_reasoning": 0,
             "reasoning_finished_count": 0,
             "reasoning_started_count": 0,
+            "reasoning_unfinished_count": 0,
+            "reasoning_finished_ratio": 0,
             "avg_reasoning_words": None,
             "avg_original_content_words": None,
             "avg_updated_content_words": None,
@@ -281,12 +283,18 @@ class ResponseReasoningInterceptor(ResponseInterceptor, PostEvalHook):
             )
 
             # Increment counters
-            if reasoning_words > 0:
+            if (
+                reasoning_words == "unknown"
+                and reasoning_info.get("reasoning_started") is True
+            ) or (isinstance(reasoning_words, int) and reasoning_words > 0):
+                # if reasoning started but not finished, or finished and we have non-zero reasoning words
                 self._reasoning_stats["responses_with_reasoning"] += 1
-            if reasoning_info.get("reasoning_started"):
+            if reasoning_info.get("reasoning_started") is True:
                 self._reasoning_stats["reasoning_started_count"] += 1
-            if reasoning_info.get("reasoning_finished"):
-                self._reasoning_stats["reasoning_finished_count"] += 1
+                if reasoning_info.get("reasoning_finished"):
+                    self._reasoning_stats["reasoning_finished_count"] += 1
+                else:
+                    self._reasoning_stats["reasoning_unfinished_count"] += 1
 
             # Update running averages
             for stat_key, value in [
@@ -340,6 +348,13 @@ class ResponseReasoningInterceptor(ResponseInterceptor, PostEvalHook):
                     updated_content_tokens
                 )
 
+            # Update ratio
+            if self._reasoning_stats["responses_with_reasoning"]:
+                self._reasoning_stats["reasoning_finished_ratio"] = (
+                    self._reasoning_stats["reasoning_finished_count"]
+                    / self._reasoning_stats["responses_with_reasoning"]
+                )
+
             # Log aggregated stats at specified interval
             if (
                 self._reasoning_stats["total_responses"]

nemo_evaluator/adapters/pipeline.py

@@ -0,0 +1,310 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Shared adapter pipeline logic used by both server and client modes."""
+
+from typing import List
+
+from nemo_evaluator.adapters.adapter_config import AdapterConfig
+from nemo_evaluator.adapters.registry import InterceptorRegistry
+from nemo_evaluator.adapters.types import (
+    AdapterGlobalContext,
+    AdapterRequest,
+    AdapterResponse,
+    FatalErrorException,
+    PostEvalHook,
+    RequestInterceptor,
+    RequestToResponseInterceptor,
+    ResponseInterceptor,
+)
+from nemo_evaluator.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class AdapterPipeline:
+    """Shared adapter pipeline that processes requests/responses through interceptors.
+
+    This class encapsulates the core adapter logic that is used by both:
+    - Server mode (AdapterServer with Flask)
+    - Client mode (AdapterTransport with httpx)
+    """
+
+    def __init__(
+        self,
+        adapter_config: AdapterConfig,
+        output_dir: str,
+        model_name: str | None = None,
+    ):
+        """Initialize the adapter pipeline.
+
+        Args:
+            adapter_config: Adapter configuration with interceptors and hooks
+            output_dir: Directory for output files
+            model_name: Optional model name for logging context
+        """
+        self.adapter_config = adapter_config
+        self.output_dir = output_dir
+        self.model_name = model_name
+
+        # Initialize interceptor chain and hooks
+        self.interceptor_chain: List[
+            RequestInterceptor | RequestToResponseInterceptor | ResponseInterceptor
+        ] = []
+        self.post_eval_hooks: List[PostEvalHook] = []
+        self._post_eval_hooks_executed: bool = False
+
+        # Initialize registry and discover components
+        self.registry = InterceptorRegistry.get_instance()
+        self.registry.discover_components(
+            modules=adapter_config.discovery.modules,
+            dirs=adapter_config.discovery.dirs,
+        )
+
+        # Validate and build chains
+        self._validate_and_build_chains()
+
+    def _validate_and_build_chains(self) -> None:
+        """Validate configuration and build interceptor chains."""
+        try:
+            # Check if adapter chain is properly defined
+            self._validate_adapter_chain_definition()
+
+            # Validate interceptor order
+            self._validate_interceptor_order()
+
+            # Build the chains
+            self._build_interceptor_chain()
+            self._build_post_eval_hooks()
+
+        except Exception as e:
+            logger.error(f"Failed to build interceptor chains: {e}")
+            raise
+
+    def _validate_adapter_chain_definition(self) -> None:
+        """Validate that the adapter chain is properly defined with at least one enabled component."""
+        enabled_interceptors = [
+            ic for ic in self.adapter_config.interceptors if ic.enabled
+        ]
+        enabled_post_eval_hooks = [
+            hook for hook in self.adapter_config.post_eval_hooks if hook.enabled
+        ]
+
+        if not enabled_interceptors and not enabled_post_eval_hooks:
+            warning_msg = (
+                "Adapter pipeline cannot start: No enabled interceptors or "
+                "post-eval hooks found. The pipeline requires at least one enabled "
+                "interceptor or post-eval hook to function properly. "
+                f"Configured interceptors: "
+                f"{[ic.name for ic in self.adapter_config.interceptors]}, "
+                f"Configured post-eval hooks: "
+                f"{[hook.name for hook in self.adapter_config.post_eval_hooks]}"
+            )
+            logger.warning(warning_msg)
+            raise RuntimeError(warning_msg)
+
+    def _validate_interceptor_order(self) -> None:
+        """Validate that the configured interceptor list follows the correct stage order.
+
+        The order must be: Request -> RequestToResponse -> Response
+        """
+        # Define stage hierarchy and allowed transitions
+        STAGE_ORDER = ["request", "request_to_response", "response"]
+        current_stage_idx = 0
+
+        for interceptor_config in self.adapter_config.interceptors:
+            if not interceptor_config.enabled:
+                continue
+
+            metadata = self.registry.get_metadata(interceptor_config.name)
+            if metadata is None:
+                raise ValueError(f"Unknown interceptor: {interceptor_config.name}")
+
+            # Determine the stage of this interceptor
+            if metadata.supports_request_to_response_interception():
+                interceptor_stage = "request_to_response"
+            elif metadata.supports_request_interception():
+                interceptor_stage = "request"
+            elif metadata.supports_response_interception():
+                interceptor_stage = "response"
+            else:
+                raise ValueError(
+                    f"Interceptor {interceptor_config.name} doesn't implement any known interface"
+                )
+
+            # Find the stage index
+            try:
+                stage_idx = STAGE_ORDER.index(interceptor_stage)
+            except ValueError:
+                raise ValueError(f"Unknown stage: {interceptor_stage}")
+
+            # Validate progression: can only move forward or stay at same stage
+            if stage_idx < current_stage_idx:
+                raise ValueError(
+                    f"Invalid stage order: interceptor {interceptor_config.name} (stage: {interceptor_stage}) "
+                    f"appears after {STAGE_ORDER[current_stage_idx]} stage. "
+                    f"Expected order: Request -> RequestToResponse -> Response"
+                )
+
+            # Update current stage if we've moved forward
+            current_stage_idx = max(current_stage_idx, stage_idx)
+
+    def _build_interceptor_chain(self) -> None:
+        """Build interceptor chain from validated configuration."""
+        self.interceptor_chain = []
+        for interceptor_config in self.adapter_config.interceptors:
+            if interceptor_config.enabled:
+                interceptor = self.registry._get_or_create_instance(
+                    interceptor_config.name,
+                    interceptor_config.config,
+                )
+                self.interceptor_chain.append(interceptor)
+
+        logger.info(
+            "Built interceptor chain",
+            interceptors=[type(i).__name__ for i in self.interceptor_chain],
+        )
+
+    def _build_post_eval_hooks(self) -> None:
+        """Build post-evaluation hooks from validated configuration."""
+        self.post_eval_hooks = []
+
+        # Add configured post-eval hooks
+        for hook_config in self.adapter_config.post_eval_hooks:
+            if hook_config.enabled:
+                hook = self.registry._get_or_create_instance(
+                    hook_config.name, hook_config.config
+                )
+                self.post_eval_hooks.append(hook)
+
+        # Also add interceptors that implement PostEvalHook
+        for interceptor in self.interceptor_chain:
+            if hasattr(interceptor, "post_eval_hook") and callable(
+                getattr(interceptor, "post_eval_hook")
+            ):
+                self.post_eval_hooks.append(interceptor)
+
+        logger.info(
+            "Built post-eval hooks",
+            hooks=[type(h).__name__ for h in self.post_eval_hooks],
+        )
+
+    def process_request(
+        self, adapter_request: AdapterRequest, global_context: AdapterGlobalContext
+    ) -> tuple[AdapterRequest, AdapterResponse | None]:
+        """Process request through the interceptor chain.
+
+        Args:
+            adapter_request: The request to process
+            global_context: Global context for the request
+
+        Returns:
+            Tuple of (modified_request, optional_response)
+            - If an interceptor returns a response, it's returned as the second element
+            - Otherwise, the second element is None and the first is the modified request
+        """
+        current_request = adapter_request
+        request_logger = get_logger()
+
+        for interceptor in self.interceptor_chain:
+            try:
+                if isinstance(
+                    interceptor, (RequestInterceptor, RequestToResponseInterceptor)
+                ):
+                    result = interceptor.intercept_request(
+                        current_request, global_context
+                    )
+
+                    # If interceptor returns a response, we're done with request processing
+                    if isinstance(result, AdapterResponse):
+                        return current_request, result
+                    else:
+                        current_request = result
+                else:
+                    # This is a ResponseInterceptor, skip in request phase
+                    continue
+
+            except FatalErrorException:
+                # Re-raise fatal errors
+                raise
+            except Exception as e:
+                request_logger.error(
+                    f"Request interceptor {type(interceptor).__name__} failed: {e}"
+                )
+                # Continue with next interceptor
+                continue
+
+        return current_request, None
+
+    def process_response(
+        self, adapter_response: AdapterResponse, global_context: AdapterGlobalContext
+    ) -> AdapterResponse:
+        """Process response through the interceptor chain (in reverse order).
+
+        Args:
+            adapter_response: The response to process
+            global_context: Global context for the response
+
+        Returns:
+            Modified response after processing through all response interceptors
+        """
+        current_response = adapter_response
+        request_logger = get_logger()
+
+        for interceptor in reversed(self.interceptor_chain):
+            try:
+                if isinstance(interceptor, ResponseInterceptor):
+                    current_response = interceptor.intercept_response(
+                        current_response, global_context
+                    )
+            except FatalErrorException:
+                # Re-raise fatal errors
+                raise
+            except Exception as e:
+                request_logger.error(
+                    f"Response interceptor {type(interceptor).__name__} failed: {e}"
+                )
+                # Continue with next interceptor
+                continue
+
+        return current_response
+
+    def run_post_eval_hooks(self, url: str = "") -> None:
+        """Run all configured post-evaluation hooks.
+
+        Args:
+            url: Optional URL for global context (not always relevant)
+        """
+        if self._post_eval_hooks_executed:
+            logger.warning("Post-eval hooks have already been executed, skipping")
+            return
+
+        global_context = AdapterGlobalContext(
+            output_dir=self.output_dir,
+            url=url,
+            model_name=self.model_name,
+        )
+
+        for hook in self.post_eval_hooks:
+            try:
+                hook.post_eval_hook(global_context)
+                logger.info(f"Successfully ran post-eval hook: {type(hook).__name__}")
+            except Exception as e:
+                logger.error(f"Post-eval hook {type(hook).__name__} failed: {e}")
+                # Continue with other hooks
+                continue
+
+        self._post_eval_hooks_executed = True
+        logger.info("Post-eval hooks execution completed")