opik 1.6.4__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/metrics/llm_judges/factuality/parser.py

@@ -0,0 +1,35 @@
+import logging
+import opik.logging_messages as logging_messages
+import opik.exceptions as exceptions
+from opik.evaluation.metrics import score_result
+from opik.evaluation.metrics.llm_judges import parsing_helpers
+
+LOGGER = logging.getLogger(__name__)
+
+
+def parse_model_output(content: str, name: str) -> score_result.ScoreResult:
+    try:
+        list_content = parsing_helpers.extract_json_content_or_raise(content)
+
+        reason = ""
+        score = 0.0
+
+        for claim in list_content:
+            claim_score = float(claim["score"])
+
+            if not (0.0 <= claim_score <= 1.0):
+                raise exceptions.MetricComputationError(
+                    f"Factuality score must be between 0.0 and 1.0, got {claim_score}"
+                )
+
+            score += claim_score
+            reason += claim["reason"] + "\n"
+
+        score /= len(list_content)
+
+        return score_result.ScoreResult(name=name, value=score, reason=reason)
+    except Exception as e:
+        LOGGER.error(f"Failed to parse model output: {e}", exc_info=True)
+        raise exceptions.MetricComputationError(
+            logging_messages.FACTUALITY_SCORE_CALC_FAILED
+        )

opik/evaluation/metrics/llm_judges/factuality/template.py

@@ -46,7 +46,7 @@ def generate_query(
 ) -> str:
     examples_str = "\n\n".join(
         [
-            f"Example {i+1}:\n"
+            f"Example {i + 1}:\n"
             f"Input: {example['input']}\n"
             f"Output: {example['output']}\n"
             f"Contexts: {example['context']}\n"

opik/evaluation/metrics/llm_judges/g_eval/__init__.py

@@ -0,0 +1,5 @@
+"""Public exports for the GEval metric package."""
+
+from .metric import GEval
+
+__all__ = ["GEval"]

opik/evaluation/metrics/llm_judges/g_eval/metric.py

@@ -1,17 +1,13 @@
-import math
-from functools import cached_property
-from typing import Any, Optional, TYPE_CHECKING, Union
+from collections import OrderedDict
+from threading import Lock
+from typing import Any, Dict, Optional, Tuple, Union
 import pydantic
-import json
-
-if TYPE_CHECKING:
-    from litellm.types.utils import ModelResponse
 
 from opik.evaluation.metrics import base_metric, score_result
 from opik.evaluation.models import base_model, models_factory
-from opik.logging_messages import GEVAL_SCORE_CALC_FAILED
-from .template import G_EVAL_COT_TEMPLATE, G_EVAL_QUERY_TEMPLATE
-from opik import exceptions
+from opik.evaluation import models
+from . import template, parser
+from .presets import GEVAL_PRESETS
 
 
 class GEvalScoreFormat(pydantic.BaseModel):
@@ -19,56 +15,175 @@ class GEvalScoreFormat(pydantic.BaseModel):
     reason: str
 
 
+def _freeze_for_cache(value: Any) -> Any:
+    """Convert nested structures into hashable representations for caching."""
+
+    if isinstance(value, dict):
+        return tuple(
+            sorted((key, _freeze_for_cache(val)) for key, val in value.items())
+        )
+    if isinstance(value, (list, tuple)):
+        return tuple(_freeze_for_cache(item) for item in value)
+    if isinstance(value, set):
+        return tuple(sorted(_freeze_for_cache(item) for item in value))
+    return value
+
+
 class GEval(base_metric.BaseMetric):
+    """
+    Generalised evaluation metric that prompts an LLM to grade another LLM output.
+
+    GEval builds a reusable chain-of-thought using the provided
+    ``task_introduction`` and ``evaluation_criteria`` prompts, then requests a
+    final score and rationale for each evaluated output.
+
+    Args:
+        task_introduction: Instruction describing the evaluator's persona/purpose.
+        evaluation_criteria: Detailed rubric presented to the evaluator.
+        model: Optional model identifier or ``OpikBaseModel`` for the judge.
+        name: Display name for the metric result. Defaults to ``"g_eval_metric"``.
+        track: Whether to automatically track metric results. Defaults to ``True``.
+        project_name: Optional tracking project name.
+        temperature: Sampling temperature forwarded to the judge model.
+        seed: Optional seed for reproducible generation (if supported by the model).
+
+    Example:
+        >>> from opik.evaluation.metrics.llm_judges.g_eval.metric import GEval
+        >>> metric = GEval(
+        ...     task_introduction="You evaluate politeness of responses.",
+        ...     evaluation_criteria="Score from 1 (rude) to 5 (very polite).",
+        ...     model="gpt-4",
+        ... )
+        >>> result = metric.score(output="Thanks so much for your help!")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.9
+    """
+
+    _CHAIN_OF_THOUGHT_CACHE: "OrderedDict[Tuple[str, str, str, Any], str]" = (
+        OrderedDict()
+    )
+    _CHAIN_OF_THOUGHT_LOCK: Lock = Lock()
+    _MAX_CHAIN_OF_THOUGHT_CACHE = 128
+
     def __init__(
         self,
         task_introduction: str,
         evaluation_criteria: str,
-        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        model: Optional[Union[str, models.base_model.OpikBaseModel]] = None,
         name: str = "g_eval_metric",
         track: bool = True,
+        project_name: Optional[str] = None,
+        temperature: float = 0.0,
+        seed: Optional[int] = None,
     ):
-        """
-        A metric that evaluates an LLM output based on chain-of-thought built with the evaluation criteria provided
-        by the user.
-
-        For more details see the original paper: https://arxiv.org/pdf/2303.16634
-
-        Args:
-            task_introduction: An instruction for LLM used to generate an evaluation chain-of-thought and in evaluation call itself.
-                `opik.evaluation.models.LiteLLMChatModel` is used by default.
-            evaluation_criteria: The main task for G-Eval metric written in human language.
-            model: The LLM to use for evaluation. Can be a string (model name) or an `opik.evaluation.models.OpikBaseModel` subclass instance.
-            name: The name of the metric.
-            track: Whether to track the metric. Defaults to True.
-        """
         super().__init__(
             name=name,
             track=track,
+            project_name=project_name,
         )
-        self._init_model(model)
-
         self.task_introduction = task_introduction
         self.evaluation_criteria = evaluation_criteria
+        self._seed = seed
+
+        self._log_probs_supported = False
+
+        self._init_model(model, temperature=temperature)
 
-    @cached_property
     def llm_chain_of_thought(self) -> str:
-        prompt = G_EVAL_COT_TEMPLATE.format(
+        cache_key = self._chain_of_thought_cache_key()
+        cached = self._get_cached_chain_of_thought(cache_key)
+        if cached is not None:
+            return cached
+
+        prompt = template.G_EVAL_COT_TEMPLATE.format(
             task_introduction=self.task_introduction,
             evaluation_criteria=self.evaluation_criteria,
         )
-        return self._model.generate_string(input=prompt)
+        generated = self._model.generate_string(input=prompt)
+        self._store_chain_of_thought(cache_key, generated)
+        return generated
+
+    async def allm_chain_of_thought(self) -> str:
+        cache_key = self._chain_of_thought_cache_key()
+        cached = self._get_cached_chain_of_thought(cache_key)
+        if cached is not None:
+            return cached
+
+        prompt = template.G_EVAL_COT_TEMPLATE.format(
+            task_introduction=self.task_introduction,
+            evaluation_criteria=self.evaluation_criteria,
+        )
+        generated = await self._model.agenerate_string(input=prompt)
+        self._store_chain_of_thought(cache_key, generated)
+        return generated
 
     def _init_model(
-        self, model: Optional[Union[str, base_model.OpikBaseModel]]
+        self, model: Optional[Union[str, base_model.OpikBaseModel]], temperature: float
     ) -> None:
         if isinstance(model, base_model.OpikBaseModel):
             self._model = model
         else:
-            self._model = models_factory.get(
-                model_name=model,
-                must_support_arguments=["logprobs", "top_logprobs"],
-            )
+            model_kwargs = {"temperature": temperature}
+            if self._seed is not None:
+                model_kwargs["seed"] = self._seed
+
+            self._model = models_factory.get(model_name=model, **model_kwargs)
+
+        if (
+            hasattr(self._model, "supported_params")
+            and "logprobs" in self._model.supported_params
+            and "top_logprobs" in self._model.supported_params
+        ):
+            self._log_probs_supported = True
+
+    @classmethod
+    def _get_cached_chain_of_thought(
+        cls, cache_key: Tuple[str, str, str, Any]
+    ) -> Optional[str]:
+        with cls._CHAIN_OF_THOUGHT_LOCK:
+            value = cls._CHAIN_OF_THOUGHT_CACHE.get(cache_key)
+            if value is not None:
+                cls._CHAIN_OF_THOUGHT_CACHE.move_to_end(cache_key)
+            return value
+
+    @classmethod
+    def _store_chain_of_thought(
+        cls, cache_key: Tuple[str, str, str, Any], value: str
+    ) -> None:
+        with cls._CHAIN_OF_THOUGHT_LOCK:
+            existing = cls._CHAIN_OF_THOUGHT_CACHE.get(cache_key)
+            if existing is not None:
+                cls._CHAIN_OF_THOUGHT_CACHE.move_to_end(cache_key)
+                return
+            cls._CHAIN_OF_THOUGHT_CACHE[cache_key] = value
+            cls._CHAIN_OF_THOUGHT_CACHE.move_to_end(cache_key)
+            while len(cls._CHAIN_OF_THOUGHT_CACHE) > cls._MAX_CHAIN_OF_THOUGHT_CACHE:
+                cls._CHAIN_OF_THOUGHT_CACHE.popitem(last=False)
+
+    def _chain_of_thought_cache_key(self) -> Tuple[str, str, str, Any]:
+        model_name = getattr(self._model, "model_name", "unknown")
+        return (
+            self.task_introduction,
+            self.evaluation_criteria,
+            model_name,
+            self._model_cache_fingerprint(),
+        )
+
+    def _model_cache_fingerprint(self) -> Any:
+        fingerprint_candidate = getattr(self._model, "cache_fingerprint", None)
+        if callable(fingerprint_candidate):
+            try:
+                fingerprint = fingerprint_candidate()
+            except Exception:
+                fingerprint = None
+            else:
+                return _freeze_for_cache(fingerprint)
+
+        completion_kwargs = getattr(self._model, "_completion_kwargs", None)
+        if isinstance(completion_kwargs, dict):
+            return _freeze_for_cache(completion_kwargs)
+
+        return id(self._model)
 
     def score(
         self,
@@ -86,10 +201,10 @@ class GEval(base_metric.BaseMetric):
             score_result.ScoreResult: A ScoreResult object containing the G-Eval score
             (between 0.0 and 1.0) and a reason for the score.
         """
-        llm_query = G_EVAL_QUERY_TEMPLATE.format(
+        llm_query = template.G_EVAL_QUERY_TEMPLATE.format(
             task_introduction=self.task_introduction,
             evaluation_criteria=self.evaluation_criteria,
-            chain_of_thought=self.llm_chain_of_thought,
+            chain_of_thought=self.llm_chain_of_thought(),
             input=output,
         )
 
@@ -100,33 +215,44 @@ class GEval(base_metric.BaseMetric):
             },
         ]
 
-        model_output = self._model.generate_provider_response(
-            messages=request,
-            logprobs=True,
-            top_logprobs=20,
-            response_format=GEvalScoreFormat,
+        if isinstance(self._model, models.LiteLLMChatModel):
+            provider_kwargs: Dict[str, Any] = {
+                "response_format": GEvalScoreFormat,
+            }
+            if self._log_probs_supported:
+                provider_kwargs["logprobs"] = True
+                provider_kwargs["top_logprobs"] = 20
+
+            with base_model.get_provider_response(
+                model_provider=self._model,
+                messages=request,
+                **provider_kwargs,
+            ) as model_output:
+                return parser.parse_litellm_model_output(
+                    content=model_output,
+                    name=self.name,
+                    log_probs_supported=self._log_probs_supported,
+                )
+
+        model_output_string = self._model.generate_string(
+            input=llm_query, response_format=GEvalScoreFormat
         )
 
-        return self._parse_model_output(model_output)
+        return parser.parse_model_output_string(model_output_string, self.name)
 
     async def ascore(
-        self, output: str, **ignored_kwargs: Any
+        self,
+        output: str,
+        **ignored_kwargs: Any,
     ) -> score_result.ScoreResult:
         """
-        Calculate the G-Eval score for the given LLM's output.
-
-        Args:
-            output: The LLM's output to evaluate.
-            **ignored_kwargs: Additional keyword arguments that are ignored.
-
-        Returns:
-            score_result.ScoreResult: A ScoreResult object containing the G-Eval score
-            (between 0.0 and 1.0) and a reason for the score.
+        Async variant of :meth:`score`, evaluating the provided LLM output using
+        the configured judge model and returning a ``ScoreResult``.
         """
-        llm_query = G_EVAL_QUERY_TEMPLATE.format(
+        llm_query = template.G_EVAL_QUERY_TEMPLATE.format(
             task_introduction=self.task_introduction,
             evaluation_criteria=self.evaluation_criteria,
-            chain_of_thought=self.llm_chain_of_thought,
+            chain_of_thought=await self.allm_chain_of_thought(),
             input=output,
         )
 
@@ -137,69 +263,74 @@ class GEval(base_metric.BaseMetric):
             },
         ]
 
-        model_output = await self._model.agenerate_provider_response(
-            messages=request,
-            logprobs=True,
-            top_logprobs=20,
-            response_format=GEvalScoreFormat,
+        if isinstance(self._model, models.LiteLLMChatModel):
+            provider_kwargs: Dict[str, Any] = {
+                "response_format": GEvalScoreFormat,
+            }
+            if self._log_probs_supported:
+                provider_kwargs["logprobs"] = True
+                provider_kwargs["top_logprobs"] = 20
+
+            async with base_model.aget_provider_response(
+                model_provider=self._model,
+                messages=request,
+                **provider_kwargs,
+            ) as model_output:
+                return parser.parse_litellm_model_output(
+                    content=model_output,
+                    name=self.name,
+                    log_probs_supported=self._log_probs_supported,
+                )
+
+        model_output_string = await self._model.agenerate_string(
+            input=llm_query, response_format=GEvalScoreFormat
         )
 
-        return self._parse_model_output(model_output)
-
-    def _parse_model_output(self, content: "ModelResponse") -> score_result.ScoreResult:
-        """
-        This method computes the final score based on the model's response. The model's response is a dictionary
-        with a `score` key and a `reason` key. The prompt template also specifies that the score should be an integer
-        between 0 and 10.
-
-        In order to make the score computation more robust, we look at the top logprobs of the score token and compute
-        a weighted average of the scores. Since we try to enforce the format of the model's response, we can assume that
-        the score token is always the fourth token in the response (first token is `{"`, followed by `score` and `":`).
-        """
-        try:
-            # Compute score using top logprobs
-            score_token_position = 3
-            top_score_logprobs = content.choices[0].model_extra["logprobs"]["content"][
-                score_token_position
-            ]["top_logprobs"]
+        return parser.parse_model_output_string(model_output_string, self.name)
 
-            linear_probs_sum = 0.0
-            weighted_score_sum = 0.0
 
-            for token_info in top_score_logprobs:
-                # litellm in v1.60.2 (or earlier) started provide logprobes
-                # as pydantic model, not just dict
-                # we will convert model to dict to provide backward compatability
-                if not isinstance(token_info, dict):
-                    token_info = token_info.model_dump()
+class GEvalPreset(GEval):
+    """
+    Pre-configured GEval variant with author-provided prompt templates.
 
-                # if not a number
-                if not token_info["token"].isdecimal():
-                    continue
+    Args:
+        preset: Key name from ``GEVAL_PRESETS`` describing the evaluation rubric.
+        model: Optional model identifier or ``OpikBaseModel`` instance.
+        track: Whether to automatically track metric results. Defaults to ``True``.
+        project_name: Optional tracking project name.
+        temperature: Sampling temperature forwarded to the judge model.
+        name: Optional override for the metric name (defaults to preset name).
 
-                score = int(token_info["token"])
+    Example:
+        >>> from opik.evaluation.metrics.llm_judges.g_eval.metric import GEvalPreset
+        >>> metric = GEvalPreset(preset="qa_relevance", model="gpt-4")
+        >>> result = metric.score(output="Answer addresses the user's question.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.85
+    """
 
-                # if score value not in scale
-                if not 0 <= score <= 10:
-                    continue
-
-                log_prob = token_info["logprob"]
-                linear_prob = math.exp(log_prob)
-
-                linear_probs_sum += linear_prob
-                weighted_score_sum += linear_prob * score
-
-            final_score: float = weighted_score_sum / linear_probs_sum / 10
-
-            if not (0.0 <= final_score <= 1.0):
-                raise ValueError
-
-            # Get the reason
-            reason = json.loads(content.choices[0].message.content)["reason"]
+    def __init__(
+        self,
+        preset: str,
+        model: Optional[Union[str, models.base_model.OpikBaseModel]] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        temperature: float = 0.0,
+        name: Optional[str] = None,
+    ):
+        try:
+            definition = GEVAL_PRESETS[preset]
+        except KeyError as error:
+            raise ValueError(
+                f"Unknown GEval preset '{preset}'. Available presets: {list(GEVAL_PRESETS)}"
+            ) from error
 
-            # Return the score and the reason
-            return score_result.ScoreResult(
-                name=self.name, value=final_score, reason=reason
-            )
-        except Exception:
-            raise exceptions.MetricComputationError(GEVAL_SCORE_CALC_FAILED)
+        super().__init__(
+            task_introduction=definition.task_introduction,
+            evaluation_criteria=definition.evaluation_criteria,
+            model=model,
+            name=name or definition.name,
+            track=track,
+            project_name=project_name,
+            temperature=temperature,
+        )

opik/evaluation/metrics/llm_judges/g_eval/parser.py

@@ -0,0 +1,161 @@
+import logging
+import json
+import math
+from typing import Any, Dict, TYPE_CHECKING
+import opik.exceptions as exceptions
+from opik.evaluation.metrics import score_result
+from opik.evaluation.metrics.llm_judges import parsing_helpers
+from opik.logging_messages import GEVAL_SCORE_CALC_FAILED
+
+if TYPE_CHECKING:  # TODO: Daniel check if this is needed
+    from litellm.types.utils import ModelResponse as LiteLLMModelResponse
+
+LOGGER = logging.getLogger(__name__)
+
+
+def parse_model_output_string(
+    content: str, metric_name: str
+) -> score_result.ScoreResult:
+    try:
+        dict_content = parsing_helpers.extract_json_content_or_raise(content)
+
+        score_raw = float(dict_content["score"])
+
+        if not 0 <= score_raw <= 10:
+            raise ValueError(
+                f"LLM returned score outside of [0, 10] range: {score_raw}"
+            )
+
+        normalised_score = score_raw / 10
+
+        reason = str(dict_content["reason"])
+
+        return score_result.ScoreResult(
+            name=metric_name,
+            value=normalised_score,
+            reason=reason,
+        )
+    except Exception as exception:
+        LOGGER.error(f"Failed to parse model output: {exception}", exc_info=True)
+        raise exceptions.MetricComputationError(GEVAL_SCORE_CALC_FAILED) from exception
+
+
+def parse_litellm_model_output(
+    content: "LiteLLMModelResponse", name: str, log_probs_supported: bool
+) -> score_result.ScoreResult:
+    """
+    This method computes the final score based on the model's response. The model's response is a dictionary
+    with a `score` key and a `reason` key. The prompt template also specifies that the score should be an integer
+    between 0 and 10.
+
+    In order to make the score computation more robust, we look at the top logprobs of the score token and compute
+    a weighted average of the scores. Since we try to enforce the format of the model's response, we can assume that
+    the score token is always the fourth token in the response (first token is `{"`, followed by `score` and `":`).
+    """
+    try:
+        choice_dict = _normalise_first_choice(content)
+
+        if not log_probs_supported:
+            return _extract_score_from_text_content(choice_dict, name=name)
+
+        log_probs = _to_dict(choice_dict.get("logprobs"))
+        entries = log_probs.get("content") or []
+        score_token_position = 3
+        if len(entries) <= score_token_position:
+            return _extract_score_from_text_content(choice_dict, name=name)
+
+        entry_dict = _to_dict(entries[score_token_position])
+        top_logprobs = entry_dict.get("top_logprobs") or []
+        token_candidate = str(entry_dict.get("token", ""))
+
+        linear_probs_sum = 0.0
+        weighted_score_sum = 0.0
+
+        for candidate in top_logprobs:
+            token_info = _to_dict(candidate)
+            token_str = str(token_info.get("token", ""))
+            if not token_str.isdecimal():
+                continue
+
+            score = int(token_str)
+            if not 0 <= score <= 10:
+                continue
+
+            log_prob = token_info.get("logprob")
+            if log_prob is None:
+                continue
+
+            linear_prob = math.exp(float(log_prob))
+            linear_probs_sum += linear_prob
+            weighted_score_sum += linear_prob * score
+
+        if linear_probs_sum != 0.0:
+            final_score: float = weighted_score_sum / linear_probs_sum / 10
+        else:
+            if not token_candidate.isdecimal():
+                raise exceptions.MetricComputationError(GEVAL_SCORE_CALC_FAILED)
+            final_score = int(token_candidate) / 10
+
+        if not (0.0 <= final_score <= 1.0):
+            raise ValueError(
+                f"Failed to compute final score from log_probs, the value is out of [0, 1] range: {final_score}"
+            )
+
+        reason_data = json.loads(_extract_message_content(choice_dict))
+        reason = reason_data["reason"]
+        return score_result.ScoreResult(name=name, value=final_score, reason=reason)
+    except Exception as exception:
+        LOGGER.error(f"Failed to parse model output: {exception}", exc_info=True)
+        raise exceptions.MetricComputationError(GEVAL_SCORE_CALC_FAILED) from exception
+
+
+def _extract_score_from_text_content(
+    choice: Dict[str, Any], name: str
+) -> score_result.ScoreResult:
+    text_content = _extract_message_content(choice)
+    return parse_model_output_string(text_content, name)
+
+
+def _extract_message_content(choice: Dict[str, Any]) -> str:
+    message = choice.get("message")
+    if isinstance(message, dict):
+        content = message.get("content")
+    else:
+        content = getattr(message, "content", None)
+
+    if not isinstance(content, str):
+        raise ValueError("LLM response is missing textual content")
+
+    return content
+
+
+def _normalise_choice(choice: Any) -> Dict[str, Any]:
+    choice_dict = _to_dict(choice)
+    if choice_dict:
+        return choice_dict
+    return {
+        "message": getattr(choice, "message", None),
+        "logprobs": getattr(choice, "logprobs", None),
+    }
+
+
+def _normalise_first_choice(response: Any) -> Dict[str, Any]:
+    choices = getattr(response, "choices", None)
+    if not isinstance(choices, list) or not choices:
+        raise exceptions.MetricComputationError(
+            "LLM response did not contain any choices to parse."
+        )
+    return _normalise_choice(choices[0])
+
+
+def _to_dict(value: Any) -> Dict[str, Any]:
+    if isinstance(value, dict):
+        return value
+    if hasattr(value, "model_dump") and callable(value.model_dump):
+        try:
+            return value.model_dump()
+        except TypeError:
+            pass
+    if hasattr(value, "__dict__"):
+        return dict(value.__dict__)
+    return {}