opik 1.6.4__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/models/litellm/litellm_chat_model.py

@@ -2,48 +2,89 @@ import importlib.metadata
 import logging
 import warnings
 from functools import cached_property
-from typing import Any, Dict, List, Optional, Set, TYPE_CHECKING
+from typing import Any, Dict, List, Optional, Set, TYPE_CHECKING, Type
+import pydantic
+import tenacity
 
 if TYPE_CHECKING:
     from litellm.types.utils import ModelResponse
 
-from opik import semantic_version
+import opik.semantic_version as semantic_version
+import opik.integrations.litellm as litellm_integration
+import opik.config as opik_config
 
 from .. import base_model
-from . import opik_monitor, warning_filters
+from . import warning_filters, util
+from opik import exceptions
 
 LOGGER = logging.getLogger(__name__)
 
 
+def _log_warning(message: str, *args: Any) -> None:
+    """Emit a warning to both this module logger and the root logger.
+
+    pytest's logging capture hooks into the root logger, while production runs use
+    the module-level logger. Logging to both keeps warnings visible in tests and at
+    runtime without duplicating call sites.
+    """
+
+    LOGGER.warning(message, *args)
+    root_logger = logging.getLogger()
+    if root_logger is not LOGGER:
+        root_logger.log(logging.WARNING, message, *args)
+
+
+def _extract_message_content(choice: Dict[str, Any]) -> Optional[str]:
+    message = choice.get("message")
+    if isinstance(message, dict):
+        content = message.get("content")
+    else:
+        content = getattr(message, "content", None)
+    if content is not None and not isinstance(content, str):
+        raise ValueError("LLM choice contains non-text content")
+    return content
+
+
+def _first_choice(response: Any) -> Dict[str, Any]:
+    choices = getattr(response, "choices", None)
+    if not isinstance(choices, list) or not choices:
+        raise exceptions.BaseLLMError(
+            "LLM response did not contain any choices to parse."
+        )
+    return util.normalise_choice(choices[0])
+
+
 class LiteLLMChatModel(base_model.OpikBaseModel):
     def __init__(
         self,
-        model_name: str = "gpt-3.5-turbo",
+        model_name: str = "gpt-5-nano",
         must_support_arguments: Optional[List[str]] = None,
         **completion_kwargs: Any,
     ) -> None:
+        import litellm
+
         """
         Initializes the base model with a given model name.
-        Wraps `litellm.completion` function.
         You can find all possible completion_kwargs parameters here: https://docs.litellm.ai/docs/completion/input.
 
         Args:
             model_name: The name of the LLM to be used.
                 This parameter will be passed to `litellm.completion(model=model_name)` so you don't need to pass
-                the `model` argument separately inside **completion_kwargs.
+                the `model` argument separately inside completion_kwargs.
             must_support_arguments: A list of openai-like arguments that the given model + provider pair must support.
                 `litellm.get_supported_openai_params(model_name)` call is used to get
                 supported arguments. If any is missing, ValueError is raised.
                 You can pass the arguments from the table: https://docs.litellm.ai/docs/completion/input#translated-openai-params
 
-            **completion_kwargs: key-value arguments to always pass additionally into `litellm.completion` function.
+            completion_kwargs: key-value arguments to always pass additionally into `litellm.completion` function.
         """
-
         super().__init__(model_name=model_name)
 
         self._check_model_name()
         self._check_must_support_arguments(must_support_arguments)
 
+        self._unsupported_warned: Set[str] = set()
+
         self._completion_kwargs: Dict[str, Any] = (
             self._remove_unnecessary_not_supported_params(completion_kwargs)
         )
@@ -54,11 +95,21 @@ class LiteLLMChatModel(base_model.OpikBaseModel):
             # Litellm has already fixed that, but it is not released yet, so this filter
             # should be removed from here soon.
             warnings.simplefilter("ignore")
-            import litellm
 
         warning_filters.add_warning_filters()
 
-        self._engine = litellm
+        config = opik_config.OpikConfig()
+
+        if config.enable_litellm_models_monitoring:
+            self._litellm_completion = litellm_integration.track_completion()(
+                litellm.completion
+            )
+            self._litellm_acompletion = litellm_integration.track_completion()(
+                litellm.acompletion
+            )
+        else:
+            self._litellm_completion = litellm.completion
+            self._litellm_acompletion = litellm.acompletion
 
     @cached_property
     def supported_params(self) -> Set[str]:
@@ -113,6 +164,7 @@ class LiteLLMChatModel(base_model.OpikBaseModel):
     ) -> Dict[str, Any]:
         filtered_params = {**params}
 
+        # Fix for impacted providers like Groq and OpenAI
         if (
             "response_format" in params
             and "response_format" not in self.supported_params
@@ -121,21 +173,74 @@ class LiteLLMChatModel(base_model.OpikBaseModel):
             LOGGER.debug(
                 "This model does not support the response_format parameter and it will be ignored."
             )
+        # NOTE: Filtering based on `supported_params` has been disabled temporarily
+        # because LiteLLM does not surface provider-specific connection fields via
+        # `get_supported_openai_params`. Dropping those kwargs breaks Azure/Groq
+        # users who rely on parameters such as `api_version` and `azure_endpoint`.
+        # The old logic is kept here commented for future restoration.
+        #
+        # for key in list(filtered_params.keys()):
+        #     if (
+        #         key not in self.supported_params
+        #         and not util.should_preserve_provider_param(key)
+        #     ):
+        #         filtered_params.pop(key)
+        #         if key not in self._unsupported_warned:
+        #             _log_warning(
+        #                 "Parameter '%s' is not supported by model %s and will be ignored.",
+        #                 key,
+        #                 self.model_name,
+        #             )
+        #             self._unsupported_warned.add(key)
+
+        util.apply_model_specific_filters(
+            model_name=self.model_name,
+            params=filtered_params,
+            already_warned=self._unsupported_warned,
+            warn=self._warn_about_unsupported_param,
+        )
 
         return filtered_params
 
-    def generate_string(self, input: str, **kwargs: Any) -> str:
+    def _warn_about_unsupported_param(self, param: str, value: Any) -> None:
+        if param in {"logprobs", "top_logprobs"}:
+            # LiteLLM warns noisily when models like gpt-5-nano don't support these
+            # fields. We already drop them gracefully, so skip logging to avoid
+            # spamming GEval users with repeated warnings.
+            return
+        if param == "temperature":
+            _log_warning(
+                "Model %s only supports temperature=1. Dropping temperature=%s.",
+                self.model_name,
+                value,
+            )
+        else:
+            _log_warning(
+                "Model %s does not support %s. Dropping the parameter.",
+                self.model_name,
+                param,
+            )
+
+    def generate_string(
+        self,
+        input: str,
+        response_format: Optional[Type[pydantic.BaseModel]] = None,
+        **kwargs: Any,
+    ) -> str:
         """
         Simplified interface to generate a string output from the model.
         You can find all possible completion_kwargs parameters here: https://docs.litellm.ai/docs/completion/input
 
         Args:
             input: The input string based on which the model will generate the output.
+            response_format: pydantic model specifying the expected output string format.
             kwargs: Additional arguments that may be used by the model for string generation.
 
         Returns:
             str: The generated string output.
         """
+        if response_format is not None:
+            kwargs["response_format"] = response_format
 
         valid_litellm_params = self._remove_unnecessary_not_supported_params(kwargs)
 
@@ -146,57 +251,80 @@ class LiteLLMChatModel(base_model.OpikBaseModel):
             },
         ]
 
-        response = self.generate_provider_response(
-            messages=request, **valid_litellm_params
-        )
-        return response.choices[0].message.content
+        with base_model.get_provider_response(
+            model_provider=self,
+            messages=request,
+            **valid_litellm_params,
+        ) as response:
+            choice = _first_choice(response)
+            content = _extract_message_content(choice)
+            return base_model.check_model_output_string(content)
 
     def generate_provider_response(
         self,
+        messages: List[Dict[str, Any]],
         **kwargs: Any,
     ) -> "ModelResponse":
         """
+        Do not use this method directly. It is intended to be used within `base_model.get_provider_response()` method.
+
         Generate a provider-specific response. Can be used to interface with
         the underlying model provider (e.g., OpenAI, Anthropic) and get raw output.
         You can find all possible input parameters here: https://docs.litellm.ai/docs/completion/input
 
         Args:
+            messages: A list of messages to be sent to the model, should be a list of dictionaries with the keys
+                "content" and "role".
             kwargs: arguments required by the provider to generate a response.
 
         Returns:
             Any: The response from the model provider, which can be of any type depending on the use case and LLM.
         """
 
-        # we need to pop messages first, and after we will check the rest params
-        messages = kwargs.pop("messages")
+        # Extract retry configuration before filtering params
+        retries = kwargs.pop("__opik_retries", 3)
+        try:
+            max_attempts = max(1, int(retries))
+        except (TypeError, ValueError):
+            max_attempts = 1
 
+        # we need to pop messages first, and after we will check the rest params
         valid_litellm_params = self._remove_unnecessary_not_supported_params(kwargs)
         all_kwargs = {**self._completion_kwargs, **valid_litellm_params}
 
-        if (
-            opik_monitor.enabled_in_config()
-            and not opik_monitor.opik_is_misconfigured()
-        ):
-            all_kwargs = opik_monitor.try_add_opik_monitoring_to_params(all_kwargs)
-
-        response = self._engine.completion(
-            model=self.model_name, messages=messages, **all_kwargs
+        retrying = tenacity.Retrying(
+            reraise=True,
+            stop=tenacity.stop_after_attempt(max_attempts),
+            wait=tenacity.wait_exponential(multiplier=0.5, min=0.5, max=8.0),
         )
 
-        return response
+        return retrying(
+            self._litellm_completion,
+            model=self.model_name,
+            messages=messages,
+            **all_kwargs,
+        )
 
-    async def agenerate_string(self, input: str, **kwargs: Any) -> str:
+    async def agenerate_string(
+        self,
+        input: str,
+        response_format: Optional[Type[pydantic.BaseModel]] = None,
+        **kwargs: Any,
+    ) -> str:
         """
         Simplified interface to generate a string output from the model. Async version.
         You can find all possible input parameters here: https://docs.litellm.ai/docs/completion/input
 
         Args:
             input: The input string based on which the model will generate the output.
+            response_format: pydantic model specifying the expected output string format.
             kwargs: Additional arguments that may be used by the model for string generation.
 
         Returns:
             str: The generated string output.
         """
+        if response_format is not None:
+            kwargs["response_format"] = response_format
 
         valid_litellm_params = self._remove_unnecessary_not_supported_params(kwargs)
 
@@ -207,35 +335,53 @@ class LiteLLMChatModel(base_model.OpikBaseModel):
             },
         ]
 
-        response = await self.agenerate_provider_response(
-            messages=request, **valid_litellm_params
-        )
-        return response.choices[0].message.content
+        async with base_model.aget_provider_response(
+            model_provider=self, messages=request, **valid_litellm_params
+        ) as response:
+            choice = _first_choice(response)
+            content = _extract_message_content(choice)
+            return base_model.check_model_output_string(content)
 
-    async def agenerate_provider_response(self, **kwargs: Any) -> "ModelResponse":
+    async def agenerate_provider_response(
+        self, messages: List[Dict[str, Any]], **kwargs: Any
+    ) -> "ModelResponse":
         """
+        Do not use this method directly. It is intended to be used within `base_model.aget_provider_response()` method.
+
         Generate a provider-specific response. Can be used to interface with
         the underlying model provider (e.g., OpenAI, Anthropic) and get raw output. Async version.
         You can find all possible input parameters here: https://docs.litellm.ai/docs/completion/input
 
         Args:
+            messages: A list of messages to be sent to the model, should be a list of dictionaries with the keys
+                "content" and "role".
             kwargs: arguments required by the provider to generate a response.
 
         Returns:
             Any: The response from the model provider, which can be of any type depending on the use case and LLM.
         """
 
-        # we need to pop messages first, and after we will check the rest params
-        messages = kwargs.pop("messages")
+        retries = kwargs.pop("__opik_retries", 3)
+        try:
+            max_attempts = max(1, int(retries))
+        except (TypeError, ValueError):
+            max_attempts = 1
 
         valid_litellm_params = self._remove_unnecessary_not_supported_params(kwargs)
         all_kwargs = {**self._completion_kwargs, **valid_litellm_params}
 
-        if opik_monitor.enabled_in_config():
-            all_kwargs = opik_monitor.try_add_opik_monitoring_to_params(all_kwargs)
-
-        response = await self._engine.acompletion(
-            model=self.model_name, messages=messages, **all_kwargs
+        retrying = tenacity.AsyncRetrying(
+            reraise=True,
+            stop=tenacity.stop_after_attempt(max_attempts),
+            wait=tenacity.wait_exponential(multiplier=0.5, min=0.5, max=8.0),
         )
 
-        return response
+        async for attempt in retrying:
+            with attempt:
+                return await self._litellm_acompletion(
+                    model=self.model_name, messages=messages, **all_kwargs
+                )
+
+        raise exceptions.BaseLLMError(
+            "Async LLM completion failed without raising an exception"
+        )  # pragma: no cover

opik/evaluation/models/litellm/opik_monitor.py

@@ -1,23 +1,24 @@
 import functools
 from typing import Any, Dict, Optional, TYPE_CHECKING
 
-from opik import config, opik_context
+import opik.config as config
+import opik.opik_context as opik_context
 
 if TYPE_CHECKING:
     import litellm
 
 
-def lazy_import_opik_logger() -> Optional["litellm.integrations.opik.opik.OpikLogger"]:
+def lazy_import_OpikLogger() -> Optional["litellm.integrations.opik.opik.OpikLogger"]:
     try:
-        from litellm.integrations.opik.opik import OpikLogger as litellm_opik_logger
-    except ImportError:
-        litellm_opik_logger = None
+        from litellm.integrations.opik.opik import OpikLogger
 
-    return litellm_opik_logger
+        return OpikLogger
+    except ImportError:
+        return None
 
 
 def try_add_opik_monitoring_to_params(params: Dict[str, Any]) -> Dict[str, Any]:
-    if lazy_import_opik_logger() is None:
+    if lazy_import_OpikLogger() is None:
         return params
 
     import litellm
@@ -52,17 +53,17 @@ def _add_span_metadata_to_params(params: Dict[str, Any]) -> Dict[str, Any]:
     if "current_span_data" in params.get("metadata", {}).get("opik", {}):
         return params
 
-    return {
-        **params,
-        "metadata": {
-            **params.get("metadata", {}),
-            "opik": {
-                **params.get("metadata", {}).get("opik", {}),
-                "current_span_data": current_span,
-                "project_name": current_span.project_name,
-            },
+    metadata = {
+        **params.get("metadata", {}),
+        "opik": {
+            **params.get("metadata", {}).get("opik", {}),
+            "current_span_data": current_span,
         },
     }
+    if current_span.project_name is not None:
+        metadata["opik"]["project_name"] = current_span.project_name
+
+    return {**params, "metadata": metadata}
 
 
 def _ensure_params_have_callback(params: Dict[str, Any]) -> Dict[str, Any]:
@@ -71,13 +72,13 @@ def _ensure_params_have_callback(params: Dict[str, Any]) -> Dict[str, Any]:
     has_global_opik_logger = False
     has_local_opik_logger = False
 
-    if opik_logger := lazy_import_opik_logger():
+    if OpikLoggerClass := lazy_import_OpikLogger():
         has_global_opik_logger = any(
-            isinstance(callback, opik_logger) for callback in litellm.callbacks
+            isinstance(callback, OpikLoggerClass) for callback in litellm.callbacks
         )
 
         has_local_opik_logger = any(
-            isinstance(callback, opik_logger)
+            isinstance(callback, OpikLoggerClass)
             for callback in params.get("success_callback", [])
         )
 
@@ -93,6 +94,8 @@ def _ensure_params_have_callback(params: Dict[str, Any]) -> Dict[str, Any]:
     }
 
 
-@functools.lru_cache
+@functools.lru_cache()
 def _callback_instance() -> "litellm.integrations.opik.opik.OpikLogger":  # type: ignore
-    return lazy_import_opik_logger()
+    OpikLoggerClass = lazy_import_OpikLogger()
+    assert OpikLoggerClass is not None
+    return OpikLoggerClass()

opik/evaluation/models/litellm/util.py

@@ -0,0 +1,125 @@
+"""Utility helpers shared across LiteLLM models."""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, Set
+
+
+def normalise_choice(choice: Any) -> Dict[str, Any]:
+    """Produce a dict view of a LiteLLM choice regardless of response type.
+
+    LiteLLM may return raw dicts, Pydantic models, or dataclasses. Normalising to a
+    dict here keeps downstream parsing logic consistent and backwards compatible with
+    older client versions.
+    """
+
+    if isinstance(choice, dict):
+        return choice
+    if hasattr(choice, "model_dump") and callable(choice.model_dump):
+        try:
+            return choice.model_dump()
+        except TypeError:
+            pass
+    normalised: Dict[str, Any] = {}
+    message = getattr(choice, "message", None)
+    if message is not None:
+        normalised["message"] = message
+    logprobs = getattr(choice, "logprobs", None)
+    if logprobs is not None:
+        normalised["logprobs"] = logprobs
+    return normalised
+
+
+def apply_model_specific_filters(
+    model_name: str,
+    params: Dict[str, Any],
+    already_warned: Set[str],
+    warn: Callable[[str, Any], None],
+) -> None:
+    """Adjust/drop params for specific model families before calling LiteLLM.
+
+    Currently handles:
+    - GPT-5: only honours temperature=1 and does not return log probabilities.
+    - DashScope Qwen: enforces constraints for logprobs / top_logprobs
+    """
+    if model_name.startswith("gpt-5"):
+        _apply_gpt5_filters(params, already_warned, warn)
+        return
+
+    if model_name.startswith("dashscope/"):
+        _apply_qwen_dashscope_filters(params, already_warned, warn)
+        return
+
+
+def _apply_gpt5_filters(
+    params: Dict[str, Any],
+    already_warned: Set[str],
+    warn: Callable[[str, Any], None],
+) -> None:
+    """Apply GPT-5 specific parameter filters.
+
+    Only honours temperature=1 and does not return log probabilities.
+    Removing those eagerly avoids provider errors while the callback surfaces a
+    one-time warning to the caller.
+    """
+
+    unsupported: list[tuple[str, Any]] = []
+
+    if "temperature" in params:
+        value = params["temperature"]
+        try:
+            numeric_value = float(value)
+        except (TypeError, ValueError):
+            numeric_value = None
+        if numeric_value is None or abs(numeric_value - 1.0) > 1e-6:
+            unsupported.append(("temperature", value))
+
+    for param in ("logprobs", "top_logprobs"):
+        if param in params:
+            unsupported.append((param, params[param]))
+
+    _drop_unsupported_params_with_warning(
+        params,
+        unsupported,
+        already_warned,
+        warn,
+    )
+
+
+def _apply_qwen_dashscope_filters(
+    params: Dict[str, Any],
+    already_warned: Set[str],
+    warn: Callable[[str, Any], None],
+) -> None:
+    """Apply Qwen/DashScope specific parameter filters.
+
+    Does not return log probabilities.
+    """
+
+    unsupported: list[tuple[str, Any]] = []
+
+    for param in ("logprobs", "top_logprobs"):
+        if param in params:
+            unsupported.append((param, params[param]))
+
+    _drop_unsupported_params_with_warning(
+        params,
+        unsupported,
+        already_warned,
+        warn,
+    )
+
+
+def _drop_unsupported_params_with_warning(
+    params: Dict[str, Any],
+    unsupported_params: list[tuple[str, Any]],
+    already_warned: Set[str],
+    warn: Callable[[str, Any], None],
+) -> None:
+    """Remove unsupported params and emit warnings once per param name."""
+    for param, value in unsupported_params:
+        params.pop(param, None)
+        if param in already_warned:
+            continue
+        warn(param, value)
+        already_warned.add(param)

opik/evaluation/models/litellm/warning_filters.py

@@ -3,8 +3,14 @@ import warnings
 from typing import Any
 
 
+_FILTERS_INSTALLED = False
+
+
 def add_warning_filters() -> None:
-    # TODO: This should be removed when we have fixed the error messages in the LiteLLM library
+    global _FILTERS_INSTALLED
+    if _FILTERS_INSTALLED:
+        return
+
     warnings.filterwarnings("ignore", message="coroutine '.*' was never awaited")
     warnings.filterwarnings(
         "ignore",
@@ -19,9 +25,15 @@ def add_warning_filters() -> None:
             )
 
     # Add filter to multiple possible loggers
-    filter = NoEventLoopFilterLiteLLM()
-    logging.getLogger("LiteLLM").addFilter(filter)
+    lite_logger = logging.getLogger("LiteLLM")
+    has_filter = any(
+        isinstance(f, NoEventLoopFilterLiteLLM) for f in lite_logger.filters
+    )
+    if not has_filter:
+        lite_logger.addFilter(NoEventLoopFilterLiteLLM())
 
     import litellm
 
-    litellm.suppress_debug_info = True  # to disable colorized prints with links to litellm whenever an LLM provider raises an error
+    litellm.suppress_debug_info = True
+
+    _FILTERS_INSTALLED = True