opik 1.6.4__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/models/base_model.py

@@ -1,12 +1,20 @@
 import abc
-from typing import Any
+import logging
+from contextlib import contextmanager, asynccontextmanager
+from typing import Any, List, Dict, Optional, Type
+import pydantic
+
+from opik import exceptions
+
+
+LOGGER = logging.getLogger(__name__)
 
 
 class OpikBaseModel(abc.ABC):
     """
     This class serves as an interface to LLMs.
 
-    If you want to implement custom LLM provider in evaluation metrics,
+    If you want to implement a custom LLM provider in evaluation metrics,
     you should inherit from this class.
     """
 
@@ -20,7 +28,12 @@ class OpikBaseModel(abc.ABC):
         self.model_name = model_name
 
     @abc.abstractmethod
-    def generate_string(self, input: str, **kwargs: Any) -> str:
+    def generate_string(
+        self,
+        input: str,
+        response_format: Optional[Type[pydantic.BaseModel]] = None,
+        **kwargs: Any,
+    ) -> str:
         """
         Simplified interface to generate a string output from the model.
 
@@ -34,44 +47,153 @@ class OpikBaseModel(abc.ABC):
         pass
 
     @abc.abstractmethod
-    async def agenerate_string(self, input: str, **kwargs: Any) -> str:
+    def generate_provider_response(
+        self, messages: List[Dict[str, Any]], **kwargs: Any
+    ) -> Any:
         """
-        Simplified interface to generate a string output from the model. Async version.
+        Do not use this method directly. It is intended to be used within `get_provider_response()` method.
+
+        Generate a provider-specific response. Can be used to interface with
+        the underlying model provider (e.g., OpenAI, Anthropic) and get raw output.
 
         Args:
-            input: The input string based on which the model will generate the output.
-            kwargs: Additional arguments that may be used by the model for string generation.
+            messages: A list of messages to be sent to the model, should be a list of dictionaries with the keys
+            kwargs: arguments required by the provider to generate a response.
 
         Returns:
-            str: The generated string output.
+            Any: The response from the model provider, which can be of any type depending on the use case and LLM.
         """
         pass
 
-    @abc.abstractmethod
-    def generate_provider_response(self, **kwargs: Any) -> Any:
+    async def agenerate_string(
+        self,
+        input: str,
+        response_format: Optional[Type[pydantic.BaseModel]] = None,
+        **kwargs: Any,
+    ) -> str:
         """
-        Generate a provider-specific response. Can be used to interface with
-        the underlying model provider (e.g., OpenAI, Anthropic) and get raw output.
+        Simplified interface to generate a string output from the model. Async version.
 
         Args:
-            kwargs: arguments required by the provider to generate a response.
+            input: The input string based on which the model will generate the output.
+            kwargs: Additional arguments that may be used by the model for string generation.
 
         Returns:
-            Any: The response from the model provider, which can be of any type depending on the use case and LLM.
+            str: The generated string output.
         """
-        pass
+        raise NotImplementedError("Async generation not implemented for this provider")
 
-    @abc.abstractmethod
-    async def agenerate_provider_response(self, **kwargs: Any) -> Any:
+    async def agenerate_provider_response(
+        self, messages: List[Dict[str, Any]], **kwargs: Any
+    ) -> Any:
         """
+        Do not use this method directly. It is intended to be used within `aget_provider_response()` method.
+
         Generate a provider-specific response. Can be used to interface with
         the underlying model provider (e.g., OpenAI, Anthropic) and get raw output.
         Async version.
 
         Args:
+            messages: A list of messages to be sent to the model, should be a list of dictionaries with the keys
+                "content" and "role".
             kwargs: arguments required by the provider to generate a response.
 
         Returns:
             Any: The response from the model provider, which can be of any type depending on the use case and LLM.
         """
-        pass
+        raise NotImplementedError("Async generation not implemented for this provider")
+
+
+@contextmanager
+def get_provider_response(
+    model_provider: OpikBaseModel, messages: List[Dict[str, Any]], **kwargs: Any
+) -> Any:
+    """
+    Provides a context manager for getting and managing the response from a
+    model provider. Ensures that errors during the interaction with the model
+    provider are handled appropriately and logged.
+
+    Args:
+        model_provider: Instance of a class derived from `OpikBaseModel`
+            responsible for interfacing with the model.
+        messages: List of dictionaries containing the messages or inputs to be
+            passed to the model.
+        **kwargs: Additional keyword arguments to customize the generation of
+            the model responses.
+
+    Yields:
+        Any: The response generated by the model provider.
+
+    Raises:
+        exceptions.BaseLLMError: If the response generation from the model provider
+            fails due to an exception.
+    """
+    try:
+        yield model_provider.generate_provider_response(messages, **kwargs)
+    except Exception as e:
+        LOGGER.error("Failed to call LLM provider, reason: %s", e)
+        raise exceptions.BaseLLMError(str(e))
+
+
+@asynccontextmanager
+async def aget_provider_response(
+    model_provider: OpikBaseModel, messages: List[Dict[str, Any]], **kwargs: Any
+) -> Any:
+    """
+    Asynchronous context manager for getting a response from a model provider.
+
+    This function asynchronously interacts with the specified `model_provider` to
+    generate a response based on the given list of `messages` and additional
+    optional keyword arguments. If an error occurs during this process, it is
+    logged, and a custom exception is raised.
+
+    Args:
+        model_provider: The model provider from which to request
+            the response.
+        messages: A list of dictionaries containing the
+            messages for the model provider to process.
+        **kwargs: Additional keyword arguments passed to the model provider's
+            response generation method.
+
+    Yields:
+        Any: The response generated asynchronously by the model provider.
+
+    Raises:
+        exceptions.BaseLLMError: If there is an error during the asynchronous
+            interaction with the model provider.
+    """
+    try:
+        response = await model_provider.agenerate_provider_response(
+            messages=messages, **kwargs
+        )
+        yield response
+    except Exception as e:
+        LOGGER.error("Failed to call LLM provider asynchronously, reason: %s", e)
+        raise exceptions.BaseLLMError(str(e))
+
+
+def check_model_output_string(output: Optional[str]) -> str:
+    """
+    Checks the output of a model and verifies that it is not None.
+
+    This function ensures that the output returned from a language model (LLM) has a valid, non-null value.
+    If the output is found to be None, an error is raised with a detailed message. This can help in
+    debugging issues related to incorrect environment configuration or missing API keys.
+
+    Args:
+        output: The output string generated by the language model to be validated.
+
+    Returns:
+        The output of the language model that was validated.
+
+    Raises:
+        exceptions.BaseLLMError: Raised if the output is evaluated to None. The error message contains suggestions to
+        verify environment configurations and check model API key availability.
+    """
+    if output is None:
+        raise exceptions.BaseLLMError(
+            "Received None as the output from the LLM. Please verify your environment configuration "
+            "and ensure that the API keys for the models in use (e.g., OPENAI_API_KEY) are set correctly."
+        )
+
+    return output

opik/evaluation/models/langchain/__init__.py

@@ -0,0 +1,3 @@
+from .langchain_chat_model import LangchainChatModel
+
+__all__ = ["LangchainChatModel"]

opik/evaluation/models/langchain/langchain_chat_model.py

@@ -0,0 +1,166 @@
+import logging
+from typing import Any, Optional, Type, List, Dict, TYPE_CHECKING
+import pydantic
+
+from . import opik_monitoring, message_converters
+from ...models import base_model
+
+if TYPE_CHECKING:
+    import langchain_core.language_models
+    import langchain_core.messages
+
+LOGGER = logging.getLogger(__name__)
+
+
+class LangchainChatModel(base_model.OpikBaseModel):
+    def __init__(
+        self,
+        chat_model: "langchain_core.language_models.BaseChatModel",
+        track: bool = True,
+    ) -> None:
+        """
+        Initializes the model with a given Langchain chat model instance.
+
+        Args:
+            chat_model: A Langchain chat model instance to wrap.
+                It is assumed that the BaseChatModel is already configured and
+                all the requirement dependencies are installed.
+            track: Whether to track the model calls.
+        """
+        model_name = _extract_model_name(chat_model)
+        super().__init__(model_name=model_name)
+
+        self._engine = chat_model
+        self._track = track
+
+    def generate_string(
+        self,
+        input: str,
+        response_format: Optional[Type[pydantic.BaseModel]] = None,
+        **kwargs: Any,
+    ) -> str:
+        """
+        Simplified interface to generate a string output from the model.
+
+        Args:
+            input: The input string based on which the model will generate the output.
+            response_format: pydantic model specifying the expected output string format.
+            kwargs: Additional arguments that may be used by the model for string generation.
+
+        Returns:
+            str: The generated string output.
+        """
+        if response_format is not None:
+            kwargs["response_format"] = response_format
+
+        request = [
+            {
+                "content": input,
+                "role": "user",
+            },
+        ]
+        with base_model.get_provider_response(
+            model_provider=self, messages=request, **kwargs
+        ) as response:
+            return base_model.check_model_output_string(response.content)
+
+    def generate_provider_response(
+        self,
+        messages: List[Dict[str, Any]],
+        **kwargs: Any,
+    ) -> "langchain_core.messages.AIMessage":
+        """
+        Do not use this method directly. It is intended to be used within `base_model.get_provider_response()` method.
+
+        Generate a provider-specific response using the Langchain model.
+
+        Args:
+            messages: A list of messages to be sent to the model, should be a list of dictionaries with the keys
+                "content" and "role".
+            kwargs: arguments required by the provider to generate a response.
+
+        Returns:
+            ModelResponse: The response from the model provider.
+        """
+        langchain_messages = message_converters.convert_to_langchain_messages(messages)
+
+        opik_monitoring.add_opik_tracer_to_params(kwargs)
+        response = self._engine.invoke(langchain_messages, **kwargs)
+
+        return response
+
+    async def agenerate_string(
+        self,
+        input: str,
+        response_format: Optional[Type[pydantic.BaseModel]] = None,
+        **kwargs: Any,
+    ) -> str:
+        """
+        Simplified interface to generate a string output from the model. Async version.
+
+        Args:
+            input: The input string based on which the model will generate the output.
+            response_format: pydantic model specifying the expected output string format.
+            kwargs: Additional arguments that may be used by the model for string generation.
+
+        Returns:
+            str: The generated string output.
+        """
+        if response_format is not None:
+            kwargs["response_format"] = response_format
+
+        request = [
+            {
+                "content": input,
+                "role": "user",
+            },
+        ]
+
+        async with base_model.aget_provider_response(
+            model_provider=self, messages=request, **kwargs
+        ) as response:
+            return base_model.check_model_output_string(response.content)
+
+    async def agenerate_provider_response(
+        self, messages: List[Dict[str, Any]], **kwargs: Any
+    ) -> "langchain_core.messages.AIMessage":
+        """
+        Do not use this method directly. It is intended to be used within `base_model.aget_provider_response()` method.
+
+        Generate a provider-specific response using the Langchain model. Async version.
+
+        Args:
+            messages: A list of messages to be sent to the model, should be a list of dictionaries with the keys
+                "content" and "role".
+            kwargs: arguments required by the provider to generate a response.
+
+        Returns:
+            ModelResponse: The response from the model provider.
+        """
+        langchain_messages = message_converters.convert_to_langchain_messages(messages)
+
+        opik_monitoring.add_opik_tracer_to_params(kwargs)
+        response = await self._engine.ainvoke(langchain_messages, **kwargs)
+
+        return response
+
+
+def _extract_model_name(
+    langchain_chat_model: "langchain_core.language_models.BaseChatModel",
+) -> str:
+    if hasattr(langchain_chat_model, "model") and isinstance(
+        langchain_chat_model.model, str
+    ):
+        return langchain_chat_model.model
+
+    if hasattr(langchain_chat_model, "model_name") and isinstance(
+        langchain_chat_model.model_name, str
+    ):
+        return langchain_chat_model.model_name
+
+    if hasattr(langchain_chat_model, "model_id") and isinstance(
+        langchain_chat_model.model_id, str
+    ):
+        return langchain_chat_model.model_id
+
+    return "unknown-model"

opik/evaluation/models/langchain/message_converters.py

@@ -0,0 +1,106 @@
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Union, cast
+
+if TYPE_CHECKING:
+    import langchain_core.messages
+
+ContentType = Union[str, List[Dict[str, Any]]]
+
+
+_ROLE_TO_MESSAGE_CLASS: Mapping[str, str] = {
+    "system": "SystemMessage",
+    "user": "HumanMessage",
+    "assistant": "AIMessage",
+    "human": "HumanMessage",
+    "ai": "AIMessage",
+}
+
+
+def convert_to_langchain_messages(
+    messages: Iterable[Mapping[str, Any]],
+) -> List["langchain_core.messages.BaseMessage"]:
+    """Convert OpenAI-style chat messages to LangChain's primitives.
+
+    Args:
+        messages: Iterable of message dictionaries in the OpenAI schema. Each
+            dictionary must include a ``role`` key and a ``content`` value that is
+            either a string or a list of content blocks (``{"type": ..., ...}``).
+
+    Returns:
+        A list of LangChain ``BaseMessage`` instances preserving the original
+        content structure.
+
+    Raises:
+        ValueError: If a message role is unsupported or required metadata is
+            missing (for example ``tool_call_id`` on ``tool`` messages).
+        TypeError: If a content payload is not a string or list.
+    """
+
+    import langchain_core.messages
+
+    role_mapping = {
+        role: getattr(langchain_core.messages, class_name)
+        for role, class_name in _ROLE_TO_MESSAGE_CLASS.items()
+    }
+
+    langchain_messages: List["langchain_core.messages.BaseMessage"] = []
+    for message in messages:
+        payload: Mapping[str, Any] = message
+
+        # messages_to_dict may wrap the payload under "data" for some message types
+        if "content" not in payload and isinstance(message.get("data"), Mapping):
+            payload = message["data"]  # type: ignore[index]
+
+        role_value = (
+            message.get("role")
+            or message.get("type")
+            or payload.get("role")
+            or payload.get("type")
+        )
+        if role_value is None:
+            raise ValueError("Message payload must include either 'role' or 'type'")
+
+        role = str(role_value).lower()
+
+        if "content" not in payload:
+            raise ValueError("Message payload must include a 'content' field")
+
+        content_raw = payload["content"]
+
+        if not isinstance(content_raw, (str, list)):
+            raise TypeError(
+                f"Unsupported message content type {type(content_raw)!r} for role {role}"
+            )
+
+        content = cast(ContentType, content_raw)
+        if role in role_mapping:
+            message_cls = role_mapping[role]
+            langchain_messages.append(message_cls(content=content))
+            continue
+
+        if role == "tool":
+            tool_call_id = payload.get("tool_call_id") or message.get("tool_call_id")
+            if not isinstance(tool_call_id, str):
+                raise ValueError("Tool messages must include a 'tool_call_id' field")
+            langchain_messages.append(
+                langchain_core.messages.ToolMessage(
+                    content=content,
+                    tool_call_id=tool_call_id,
+                )
+            )
+            continue
+
+        if role == "function":
+            name = payload.get("name") or message.get("name")
+            if not isinstance(name, str):
+                raise ValueError("Function messages must include a 'name' field")
+            langchain_messages.append(
+                langchain_core.messages.FunctionMessage(
+                    content=content,
+                    name=name,
+                )
+            )
+            continue
+
+        raise ValueError(f"Unsupported message role: {role}")
+
+    return langchain_messages

opik/evaluation/models/langchain/opik_monitoring.py

@@ -0,0 +1,23 @@
+from typing import Dict, Any
+
+
+def add_opik_tracer_to_params(params: Dict[str, Any]) -> Dict[str, Any]:
+    from opik.integrations.langchain import OpikTracer
+
+    opik_tracer = OpikTracer()
+
+    if "config" not in params:
+        params["config"] = {}
+
+    if "callbacks" not in params["config"]:
+        params["config"]["callbacks"] = [opik_tracer]
+        return params
+
+    callbacks = params["config"]["callbacks"]
+    for callback in callbacks:
+        if isinstance(callback, opik_tracer.__class__):
+            return params
+
+    callbacks.append(opik_tracer)
+
+    return params