opik 1.8.39__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/models/litellm/litellm_chat_model.py

@@ -4,28 +4,67 @@ import warnings
 from functools import cached_property
 from typing import Any, Dict, List, Optional, Set, TYPE_CHECKING, Type
 import pydantic
+import tenacity
 
 if TYPE_CHECKING:
     from litellm.types.utils import ModelResponse
 
 import opik.semantic_version as semantic_version
+import opik.integrations.litellm as litellm_integration
+import opik.config as opik_config
 
 from .. import base_model
-from . import opik_monitor, warning_filters
+from . import warning_filters, util
+from opik import exceptions
 
 LOGGER = logging.getLogger(__name__)
 
 
+def _log_warning(message: str, *args: Any) -> None:
+    """Emit a warning to both this module logger and the root logger.
+
+    pytest's logging capture hooks into the root logger, while production runs use
+    the module-level logger. Logging to both keeps warnings visible in tests and at
+    runtime without duplicating call sites.
+    """
+
+    LOGGER.warning(message, *args)
+    root_logger = logging.getLogger()
+    if root_logger is not LOGGER:
+        root_logger.log(logging.WARNING, message, *args)
+
+
+def _extract_message_content(choice: Dict[str, Any]) -> Optional[str]:
+    message = choice.get("message")
+    if isinstance(message, dict):
+        content = message.get("content")
+    else:
+        content = getattr(message, "content", None)
+    if content is not None and not isinstance(content, str):
+        raise ValueError("LLM choice contains non-text content")
+    return content
+
+
+def _first_choice(response: Any) -> Dict[str, Any]:
+    choices = getattr(response, "choices", None)
+    if not isinstance(choices, list) or not choices:
+        raise exceptions.BaseLLMError(
+            "LLM response did not contain any choices to parse."
+        )
+    return util.normalise_choice(choices[0])
+
+
 class LiteLLMChatModel(base_model.OpikBaseModel):
     def __init__(
         self,
-        model_name: str = "gpt-4o",
+        model_name: str = "gpt-5-nano",
         must_support_arguments: Optional[List[str]] = None,
         **completion_kwargs: Any,
     ) -> None:
+        import litellm
+
         """
         Initializes the base model with a given model name.
-        Wraps `litellm.completion` function.
         You can find all possible completion_kwargs parameters here: https://docs.litellm.ai/docs/completion/input.
 
         Args:
@@ -39,12 +78,13 @@ class LiteLLMChatModel(base_model.OpikBaseModel):
 
             completion_kwargs: key-value arguments to always pass additionally into `litellm.completion` function.
         """
-
         super().__init__(model_name=model_name)
 
         self._check_model_name()
         self._check_must_support_arguments(must_support_arguments)
 
+        self._unsupported_warned: Set[str] = set()
+
         self._completion_kwargs: Dict[str, Any] = (
             self._remove_unnecessary_not_supported_params(completion_kwargs)
         )
@@ -55,11 +95,21 @@ class LiteLLMChatModel(base_model.OpikBaseModel):
             # Litellm has already fixed that, but it is not released yet, so this filter
             # should be removed from here soon.
             warnings.simplefilter("ignore")
-            import litellm
 
         warning_filters.add_warning_filters()
 
-        self._engine = litellm
+        config = opik_config.OpikConfig()
+
+        if config.enable_litellm_models_monitoring:
+            self._litellm_completion = litellm_integration.track_completion()(
+                litellm.completion
+            )
+            self._litellm_acompletion = litellm_integration.track_completion()(
+                litellm.acompletion
+            )
+        else:
+            self._litellm_completion = litellm.completion
+            self._litellm_acompletion = litellm.acompletion
 
     @cached_property
     def supported_params(self) -> Set[str]:
@@ -114,6 +164,7 @@ class LiteLLMChatModel(base_model.OpikBaseModel):
     ) -> Dict[str, Any]:
         filtered_params = {**params}
 
+        # Fix for impacted providers like Groq and OpenAI
         if (
             "response_format" in params
             and "response_format" not in self.supported_params
@@ -122,9 +173,54 @@ class LiteLLMChatModel(base_model.OpikBaseModel):
             LOGGER.debug(
                 "This model does not support the response_format parameter and it will be ignored."
             )
+        # NOTE: Filtering based on `supported_params` has been disabled temporarily
+        # because LiteLLM does not surface provider-specific connection fields via
+        # `get_supported_openai_params`. Dropping those kwargs breaks Azure/Groq
+        # users who rely on parameters such as `api_version` and `azure_endpoint`.
+        # The old logic is kept here commented for future restoration.
+        #
+        # for key in list(filtered_params.keys()):
+        #     if (
+        #         key not in self.supported_params
+        #         and not util.should_preserve_provider_param(key)
+        #     ):
+        #         filtered_params.pop(key)
+        #         if key not in self._unsupported_warned:
+        #             _log_warning(
+        #                 "Parameter '%s' is not supported by model %s and will be ignored.",
+        #                 key,
+        #                 self.model_name,
+        #             )
+        #             self._unsupported_warned.add(key)
+
+        util.apply_model_specific_filters(
+            model_name=self.model_name,
+            params=filtered_params,
+            already_warned=self._unsupported_warned,
+            warn=self._warn_about_unsupported_param,
+        )
 
         return filtered_params
 
+    def _warn_about_unsupported_param(self, param: str, value: Any) -> None:
+        if param in {"logprobs", "top_logprobs"}:
+            # LiteLLM warns noisily when models like gpt-5-nano don't support these
+            # fields. We already drop them gracefully, so skip logging to avoid
+            # spamming GEval users with repeated warnings.
+            return
+        if param == "temperature":
+            _log_warning(
+                "Model %s only supports temperature=1. Dropping temperature=%s.",
+                self.model_name,
+                value,
+            )
+        else:
+            _log_warning(
+                "Model %s does not support %s. Dropping the parameter.",
+                self.model_name,
+                param,
+            )
+
     def generate_string(
         self,
         input: str,
@@ -155,10 +251,14 @@ class LiteLLMChatModel(base_model.OpikBaseModel):
             },
         ]
 
-        response = self.generate_provider_response(
-            messages=request, **valid_litellm_params
-        )
-        return response.choices[0].message.content
+        with base_model.get_provider_response(
+            model_provider=self,
+            messages=request,
+            **valid_litellm_params,
+        ) as response:
+            choice = _first_choice(response)
+            content = _extract_message_content(choice)
+            return base_model.check_model_output_string(content)
 
     def generate_provider_response(
         self,
@@ -166,6 +266,8 @@ class LiteLLMChatModel(base_model.OpikBaseModel):
         **kwargs: Any,
     ) -> "ModelResponse":
         """
+        Do not use this method directly. It is intended to be used within `base_model.get_provider_response()` method.
+
         Generate a provider-specific response. Can be used to interface with
         the underlying model provider (e.g., OpenAI, Anthropic) and get raw output.
         You can find all possible input parameters here: https://docs.litellm.ai/docs/completion/input
@@ -179,21 +281,29 @@ class LiteLLMChatModel(base_model.OpikBaseModel):
             Any: The response from the model provider, which can be of any type depending on the use case and LLM.
         """
 
+        # Extract retry configuration before filtering params
+        retries = kwargs.pop("__opik_retries", 3)
+        try:
+            max_attempts = max(1, int(retries))
+        except (TypeError, ValueError):
+            max_attempts = 1
+
         # we need to pop messages first, and after we will check the rest params
         valid_litellm_params = self._remove_unnecessary_not_supported_params(kwargs)
         all_kwargs = {**self._completion_kwargs, **valid_litellm_params}
 
-        if (
-            opik_monitor.enabled_in_config()
-            and not opik_monitor.opik_is_misconfigured()
-        ):
-            all_kwargs = opik_monitor.try_add_opik_monitoring_to_params(all_kwargs)
-
-        response = self._engine.completion(
-            model=self.model_name, messages=messages, **all_kwargs
+        retrying = tenacity.Retrying(
+            reraise=True,
+            stop=tenacity.stop_after_attempt(max_attempts),
+            wait=tenacity.wait_exponential(multiplier=0.5, min=0.5, max=8.0),
         )
 
-        return response
+        return retrying(
+            self._litellm_completion,
+            model=self.model_name,
+            messages=messages,
+            **all_kwargs,
+        )
 
     async def agenerate_string(
         self,
@@ -225,15 +335,19 @@ class LiteLLMChatModel(base_model.OpikBaseModel):
             },
         ]
 
-        response = await self.agenerate_provider_response(
-            messages=request, **valid_litellm_params
-        )
-        return response.choices[0].message.content
+        async with base_model.aget_provider_response(
+            model_provider=self, messages=request, **valid_litellm_params
+        ) as response:
+            choice = _first_choice(response)
+            content = _extract_message_content(choice)
+            return base_model.check_model_output_string(content)
 
     async def agenerate_provider_response(
         self, messages: List[Dict[str, Any]], **kwargs: Any
     ) -> "ModelResponse":
         """
+        Do not use this method directly. It is intended to be used within `base_model.aget_provider_response()` method.
+
         Generate a provider-specific response. Can be used to interface with
         the underlying model provider (e.g., OpenAI, Anthropic) and get raw output. Async version.
         You can find all possible input parameters here: https://docs.litellm.ai/docs/completion/input
@@ -247,14 +361,27 @@ class LiteLLMChatModel(base_model.OpikBaseModel):
             Any: The response from the model provider, which can be of any type depending on the use case and LLM.
         """
 
+        retries = kwargs.pop("__opik_retries", 3)
+        try:
+            max_attempts = max(1, int(retries))
+        except (TypeError, ValueError):
+            max_attempts = 1
+
         valid_litellm_params = self._remove_unnecessary_not_supported_params(kwargs)
         all_kwargs = {**self._completion_kwargs, **valid_litellm_params}
 
-        if opik_monitor.enabled_in_config():
-            all_kwargs = opik_monitor.try_add_opik_monitoring_to_params(all_kwargs)
-
-        response = await self._engine.acompletion(
-            model=self.model_name, messages=messages, **all_kwargs
+        retrying = tenacity.AsyncRetrying(
+            reraise=True,
+            stop=tenacity.stop_after_attempt(max_attempts),
+            wait=tenacity.wait_exponential(multiplier=0.5, min=0.5, max=8.0),
         )
 
-        return response
+        async for attempt in retrying:
+            with attempt:
+                return await self._litellm_acompletion(
+                    model=self.model_name, messages=messages, **all_kwargs
+                )
+
+        raise exceptions.BaseLLMError(
+            "Async LLM completion failed without raising an exception"
+        )  # pragma: no cover

opik/evaluation/models/litellm/util.py

@@ -0,0 +1,125 @@
+"""Utility helpers shared across LiteLLM models."""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, Set
+
+
+def normalise_choice(choice: Any) -> Dict[str, Any]:
+    """Produce a dict view of a LiteLLM choice regardless of response type.
+
+    LiteLLM may return raw dicts, Pydantic models, or dataclasses. Normalising to a
+    dict here keeps downstream parsing logic consistent and backwards compatible with
+    older client versions.
+    """
+
+    if isinstance(choice, dict):
+        return choice
+    if hasattr(choice, "model_dump") and callable(choice.model_dump):
+        try:
+            return choice.model_dump()
+        except TypeError:
+            pass
+    normalised: Dict[str, Any] = {}
+    message = getattr(choice, "message", None)
+    if message is not None:
+        normalised["message"] = message
+    logprobs = getattr(choice, "logprobs", None)
+    if logprobs is not None:
+        normalised["logprobs"] = logprobs
+    return normalised
+
+
+def apply_model_specific_filters(
+    model_name: str,
+    params: Dict[str, Any],
+    already_warned: Set[str],
+    warn: Callable[[str, Any], None],
+) -> None:
+    """Adjust/drop params for specific model families before calling LiteLLM.
+
+    Currently handles:
+    - GPT-5: only honours temperature=1 and does not return log probabilities.
+    - DashScope Qwen: enforces constraints for logprobs / top_logprobs
+    """
+    if model_name.startswith("gpt-5"):
+        _apply_gpt5_filters(params, already_warned, warn)
+        return
+
+    if model_name.startswith("dashscope/"):
+        _apply_qwen_dashscope_filters(params, already_warned, warn)
+        return
+
+
+def _apply_gpt5_filters(
+    params: Dict[str, Any],
+    already_warned: Set[str],
+    warn: Callable[[str, Any], None],
+) -> None:
+    """Apply GPT-5 specific parameter filters.
+
+    Only honours temperature=1 and does not return log probabilities.
+    Removing those eagerly avoids provider errors while the callback surfaces a
+    one-time warning to the caller.
+    """
+
+    unsupported: list[tuple[str, Any]] = []
+
+    if "temperature" in params:
+        value = params["temperature"]
+        try:
+            numeric_value = float(value)
+        except (TypeError, ValueError):
+            numeric_value = None
+        if numeric_value is None or abs(numeric_value - 1.0) > 1e-6:
+            unsupported.append(("temperature", value))
+
+    for param in ("logprobs", "top_logprobs"):
+        if param in params:
+            unsupported.append((param, params[param]))
+
+    _drop_unsupported_params_with_warning(
+        params,
+        unsupported,
+        already_warned,
+        warn,
+    )
+
+
+def _apply_qwen_dashscope_filters(
+    params: Dict[str, Any],
+    already_warned: Set[str],
+    warn: Callable[[str, Any], None],
+) -> None:
+    """Apply Qwen/DashScope specific parameter filters.
+
+    Does not return log probabilities.
+    """
+
+    unsupported: list[tuple[str, Any]] = []
+
+    for param in ("logprobs", "top_logprobs"):
+        if param in params:
+            unsupported.append((param, params[param]))
+
+    _drop_unsupported_params_with_warning(
+        params,
+        unsupported,
+        already_warned,
+        warn,
+    )
+
+
+def _drop_unsupported_params_with_warning(
+    params: Dict[str, Any],
+    unsupported_params: list[tuple[str, Any]],
+    already_warned: Set[str],
+    warn: Callable[[str, Any], None],
+) -> None:
+    """Remove unsupported params and emit warnings once per param name."""
+    for param, value in unsupported_params:
+        params.pop(param, None)
+        if param in already_warned:
+            continue
+        warn(param, value)
+        already_warned.add(param)

opik/evaluation/models/litellm/warning_filters.py

@@ -3,8 +3,14 @@ import warnings
 from typing import Any
 
 
+_FILTERS_INSTALLED = False
+
+
 def add_warning_filters() -> None:
-    # TODO: This should be removed when we have fixed the error messages in the LiteLLM library
+    global _FILTERS_INSTALLED
+    if _FILTERS_INSTALLED:
+        return
+
     warnings.filterwarnings("ignore", message="coroutine '.*' was never awaited")
     warnings.filterwarnings(
         "ignore",
@@ -19,9 +25,15 @@ def add_warning_filters() -> None:
             )
 
     # Add filter to multiple possible loggers
-    filter = NoEventLoopFilterLiteLLM()
-    logging.getLogger("LiteLLM").addFilter(filter)
+    lite_logger = logging.getLogger("LiteLLM")
+    has_filter = any(
+        isinstance(f, NoEventLoopFilterLiteLLM) for f in lite_logger.filters
+    )
+    if not has_filter:
+        lite_logger.addFilter(NoEventLoopFilterLiteLLM())
 
     import litellm
 
-    litellm.suppress_debug_info = True  # to disable colorized prints with links to litellm whenever an LLM provider raises an error
+    litellm.suppress_debug_info = True
+
+    _FILTERS_INSTALLED = True

opik/evaluation/models/model_capabilities.py

@@ -0,0 +1,187 @@
+"""
+Capability registry for evaluation models.
+
+The registry is designed to grow beyond vision support (e.g. audio in the future).
+"""
+
+from __future__ import annotations
+
+from typing import Callable, Dict, Iterable, Optional, Set
+
+CapabilityDetector = Callable[[str], bool]
+
+
+VISION_MODEL_PREFIXES: Set[str] = {
+    # OpenAI
+    "gpt-4-vision",
+    "gpt-4o",
+    "gpt-4o-mini",
+    "gpt-4-turbo",
+    "chatgpt-4o-latest",
+    "gpt-5-mini",
+    "gpt-4.1",
+    "gpt-4.1-mini",
+    "gpt-4.1-nano",
+    "gpt-4.1-preview",
+    # Anthropic
+    "claude-3",
+    "claude-3-5",
+    # Google
+    "gemini-1.5-pro",
+    "gemini-1.5-flash",
+    "gemini-pro-vision",
+    "gemini-2.0-flash",
+    # Meta
+    "llama-3.2-11b-vision",
+    "llama-3.2-90b-vision",
+    # Mistral
+    "pixtral",
+    # Misc
+    "qwen-vl",
+    "qwen2-vl",
+    "phi-3-vision",
+    "phi-3.5-vision",
+    "llava",
+    "cogvlm",
+    "yi-vl",
+}
+VISION_MODEL_PREFIXES = {prefix.lower() for prefix in VISION_MODEL_PREFIXES}
+VISION_MODEL_SUFFIXES: Set[str] = {"-vision", "-vl"}
+
+
+def _strip_provider_prefix(model_name: str) -> str:
+    if "/" not in model_name:
+        return model_name
+    _, suffix = model_name.split("/", 1)
+    return suffix
+
+
+def _litellm_supports_vision(model_name: str) -> bool:
+    try:
+        import litellm  # type: ignore
+
+        return litellm.supports_vision(model=model_name)
+    except Exception:
+        return False
+
+
+def vision_capability_detector(model_name: str) -> bool:
+    stripped = _strip_provider_prefix(model_name)
+    candidates = {model_name, stripped}
+    for candidate in candidates:
+        if _litellm_supports_vision(candidate):
+            return True
+        normalized = candidate.lower()
+        if any(normalized.startswith(prefix) for prefix in VISION_MODEL_PREFIXES):
+            return True
+        if any(normalized.endswith(suffix) for suffix in VISION_MODEL_SUFFIXES):
+            return True
+    return False
+
+
+def video_capability_detector(model_name: str) -> bool:
+    """
+    Heuristically determine whether a model accepts video inputs.
+
+    Providers rarely expose structured metadata for video support, so we fall back
+    to naming conventions (e.g. models whose names contain ``video`` or ``qwen``
+    + ``vl``). When those heuristics fail we delegate to the vision detector since
+    current SDK integrations treat video as an extension of multimodal/vision APIs.
+    """
+    stripped = _strip_provider_prefix(model_name)
+    candidates = {model_name, stripped}
+    for candidate in candidates:
+        normalized = candidate.lower()
+        if "video" in normalized:
+            return True
+        if "qwen" in normalized and "vl" in normalized:
+            return True
+    # TODO(opik): litellm/model metadata still treats video + image inputs the same.
+    # Fall back to the vision heuristic so we can keep this dedicated capability
+    # and tighten detection once providers expose richer metadata.
+    return vision_capability_detector(model_name)
+
+
+class ModelCapabilitiesRegistry:
+    """
+    Central registry for model capability detection.
+    """
+
+    def __init__(self) -> None:
+        self._capability_detectors: Dict[str, CapabilityDetector] = {}
+
+    def register_capability_detector(
+        self, capability: str, detector: CapabilityDetector
+    ) -> None:
+        """
+        Register a detector callable for a capability name.
+        """
+        self._capability_detectors[capability] = detector
+
+    def supports(self, capability: str, model_name: Optional[str]) -> bool:
+        """
+        Return True when the supplied model name supports the requested capability.
+        """
+        if not model_name:
+            return False
+
+        detector = self._capability_detectors.get(capability)
+        if detector is None:
+            return False
+
+        try:
+            return detector(model_name)
+        except Exception:
+            return False
+
+    def supports_vision(self, model_name: Optional[str]) -> bool:
+        """
+        Convenience wrapper for vision-capable detection.
+        """
+        return self.supports("vision", model_name)
+
+    def supports_video(self, model_name: Optional[str]) -> bool:
+        """
+        Convenience wrapper for video-capable detection.
+        """
+        return self.supports("video", model_name)
+
+    def add_vision_model(self, model_name: str) -> None:
+        # Extend the module-level registry used by vision_capability_detector
+        VISION_MODEL_PREFIXES.add(self._strip_provider_prefix(model_name).lower())
+
+    def add_vision_models(self, model_names: Iterable[str]) -> None:
+        for model_name in model_names:
+            self.add_vision_model(model_name)
+
+    def _supports_vision(self, model_name: str) -> bool:
+        return vision_capability_detector(model_name)
+
+    @staticmethod
+    def _strip_provider_prefix(model_name: str) -> str:
+        return _strip_provider_prefix(model_name)
+
+    @staticmethod
+    def _litellm_supports_vision(model_name: str) -> bool:
+        return _litellm_supports_vision(model_name)
+
+
+MODEL_CAPABILITIES_REGISTRY = ModelCapabilitiesRegistry()
+MODEL_CAPABILITIES_REGISTRY.register_capability_detector(
+    "vision", vision_capability_detector
+)
+MODEL_CAPABILITIES_REGISTRY.register_capability_detector(
+    "video", video_capability_detector
+)
+
+# Backwards compatibility shim for previous API which exposed a class with classmethods.
+ModelCapabilities = MODEL_CAPABILITIES_REGISTRY
+
+
+__all__ = [
+    "ModelCapabilitiesRegistry",
+    "MODEL_CAPABILITIES_REGISTRY",
+    "ModelCapabilities",
+    "vision_capability_detector",
+    "video_capability_detector",
+]

opik/evaluation/models/models_factory.py

@@ -1,13 +1,35 @@
-from typing import Optional, Any
+from typing import Optional, Any, Dict
 
 from .litellm import litellm_chat_model
 from . import base_model
 
-DEFAULT_GPT_MODEL_NAME = "gpt-4o"
+DEFAULT_GPT_MODEL_NAME = "gpt-5-nano"
+
+_MODEL_CACHE: Dict[Any, base_model.OpikBaseModel] = {}
+
+
+def _freeze(value: Any) -> Any:
+    if isinstance(value, dict):
+        return frozenset((k, _freeze(v)) for k, v in value.items())
+    if isinstance(value, (list, tuple)):
+        return tuple(_freeze(v) for v in value)
+    if isinstance(value, set):
+        return frozenset(_freeze(v) for v in value)
+    return value
+
+
+def _make_cache_key(model_name: str, model_kwargs: Dict[str, Any]) -> Any:
+    frozen_kwargs = frozenset((k, _freeze(v)) for k, v in model_kwargs.items())
+    return (model_name, frozen_kwargs)
 
 
 def get(model_name: Optional[str], **model_kwargs: Any) -> base_model.OpikBaseModel:
     if model_name is None:
         model_name = DEFAULT_GPT_MODEL_NAME
 
-    return litellm_chat_model.LiteLLMChatModel(model_name=model_name, **model_kwargs)
+    cache_key = _make_cache_key(model_name, model_kwargs)
+    if cache_key not in _MODEL_CACHE:
+        _MODEL_CACHE[cache_key] = litellm_chat_model.LiteLLMChatModel(
+            model_name=model_name, **model_kwargs
+        )
+    return _MODEL_CACHE[cache_key]