opik 1.6.4__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/metrics/llm_judges/g_eval_presets/qa_suite.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from typing import Optional, Union
+
+from opik.evaluation.metrics.llm_judges.g_eval import metric as g_eval_metric
+from opik.evaluation.models import base_model
+
+
+class SummarizationConsistencyJudge(g_eval_metric.GEvalPreset):
+    """
+    Score how faithful a summary is to its source content.
+
+    Args:
+        model: Optional model identifier or ``OpikBaseModel`` instance.
+        track: Whether to automatically track judge outputs. Defaults to ``True``.
+        project_name: Optional tracking project name.
+        temperature: Sampling temperature forwarded to the underlying model.
+
+    Example:
+        >>> from opik.evaluation.metrics import SummarizationConsistencyJudge
+        >>> judge = SummarizationConsistencyJudge(model="gpt-4")
+        >>> result = judge.score(output="Summary omits key fact.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.4
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        temperature: float = 0.0,
+    ) -> None:
+        super().__init__(
+            preset="summarization_consistency",
+            model=model,
+            track=track,
+            project_name=project_name,
+            temperature=temperature,
+            name="summarization_consistency_judge",
+        )
+
+
+class SummarizationCoherenceJudge(g_eval_metric.GEvalPreset):
+    """
+    Evaluate the coherence and structure of generated summaries.
+
+    Args:
+        model: Optional model identifier or ``OpikBaseModel`` instance.
+        track: Whether to automatically track judge outputs. Defaults to ``True``.
+        project_name: Optional tracking project name.
+        temperature: Sampling temperature forwarded to the underlying model.
+
+    Example:
+        >>> from opik.evaluation.metrics import SummarizationCoherenceJudge
+        >>> judge = SummarizationCoherenceJudge(model="gpt-4")
+        >>> result = judge.score(output="Summary jumps between unrelated topics.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.5
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        temperature: float = 0.0,
+    ) -> None:
+        super().__init__(
+            preset="summarization_coherence",
+            model=model,
+            track=track,
+            project_name=project_name,
+            temperature=temperature,
+            name="summarization_coherence_judge",
+        )
+
+
+class DialogueHelpfulnessJudge(g_eval_metric.GEvalPreset):
+    """
+    Judge how helpful an assistant reply is within a dialogue.
+
+    Args:
+        model: Optional model identifier or ``OpikBaseModel`` instance.
+        track: Whether to automatically track judge outputs. Defaults to ``True``.
+        project_name: Optional tracking project name.
+        temperature: Sampling temperature forwarded to the underlying model.
+
+    Example:
+        >>> from opik.evaluation.metrics import DialogueHelpfulnessJudge
+        >>> judge = DialogueHelpfulnessJudge(model="gpt-4")
+        >>> result = judge.score(output="Assistant politely refuses without help.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.3
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        temperature: float = 0.0,
+    ) -> None:
+        super().__init__(
+            preset="dialogue_helpfulness",
+            model=model,
+            track=track,
+            project_name=project_name,
+            temperature=temperature,
+            name="dialogue_helpfulness_judge",
+        )
+
+
+class QARelevanceJudge(g_eval_metric.GEvalPreset):
+    """
+    Check whether an answer directly addresses the user question.
+
+    Args:
+        model: Optional model identifier or ``OpikBaseModel`` instance.
+        track: Whether to automatically track judge outputs. Defaults to ``True``.
+        project_name: Optional tracking project name.
+        temperature: Sampling temperature forwarded to the underlying model.
+
+    Example:
+        >>> from opik.evaluation.metrics import QARelevanceJudge
+        >>> judge = QARelevanceJudge(model="gpt-4")
+        >>> result = judge.score(output="Answer rambles without addressing the ask.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.2
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        temperature: float = 0.0,
+    ) -> None:
+        super().__init__(
+            preset="qa_relevance",
+            model=model,
+            track=track,
+            project_name=project_name,
+            temperature=temperature,
+            name="qa_relevance_judge",
+        )

opik/evaluation/metrics/llm_judges/hallucination/metric.py

@@ -1,16 +1,10 @@
-import json
-import logging
 from typing import Union, Optional, List, Any
 import pydantic
 
 from opik.evaluation.models import base_model, models_factory
 from opik.evaluation.metrics import score_result, base_metric
-from opik import logging_messages
 
-from . import template
-from opik import exceptions
-
-LOGGER = logging.getLogger(__name__)
+from . import template, parser
 
 
 class HallucinationResponseFormat(pydantic.BaseModel):
@@ -31,6 +25,10 @@ class Hallucination(base_metric.BaseMetric):
         name: The name of the metric.
         few_shot_examples: A list of few-shot examples to use for hallucination detection.  If None, default examples will be used.
         track: Whether to track the metric. Defaults to True.
+        project_name: Optional project name to track the metric in for the cases when
+            there are no parent span/trace to inherit project name from.
+        seed: Optional seed value for reproducible model generation. If provided, this seed will be passed to the model for deterministic outputs.
+        temperature: Optional temperature value for model generation. If provided, this temperature will be passed to the model. If not provided, the model's default temperature will be used.
 
     Example:
         >>> from opik.evaluation.metrics import Hallucination
@@ -52,18 +50,30 @@ class Hallucination(base_metric.BaseMetric):
         name: str = "hallucination_metric",
         few_shot_examples: Optional[List[template.FewShotExampleHallucination]] = None,
         track: bool = True,
+        project_name: Optional[str] = None,
+        seed: Optional[int] = None,
+        temperature: Optional[float] = None,
     ):
-        super().__init__(name=name, track=track)
-        self._init_model(model)
+        super().__init__(name=name, track=track, project_name=project_name)
+        self._seed = seed
+        self._init_model(model, temperature=temperature)
         self.few_shot_examples = few_shot_examples
 
     def _init_model(
-        self, model: Optional[Union[str, base_model.OpikBaseModel]]
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]],
+        temperature: Optional[float],
     ) -> None:
         if isinstance(model, base_model.OpikBaseModel):
             self._model = model
         else:
-            self._model = models_factory.get(model_name=model)
+            model_kwargs = {}
+            if temperature is not None:
+                model_kwargs["temperature"] = temperature
+            if self._seed is not None:
+                model_kwargs["seed"] = self._seed
+
+            self._model = models_factory.get(model_name=model, **model_kwargs)
 
     def score(
         self,
@@ -95,7 +105,7 @@ class Hallucination(base_metric.BaseMetric):
             input=llm_query, response_format=HallucinationResponseFormat
         )
 
-        return self._parse_model_output(model_output)
+        return parser.parse_model_output(content=model_output, name=self.name)
 
     async def ascore(
         self,
@@ -127,18 +137,4 @@ class Hallucination(base_metric.BaseMetric):
             input=llm_query, response_format=HallucinationResponseFormat
         )
 
-        return self._parse_model_output(model_output)
-
-    def _parse_model_output(self, content: str) -> score_result.ScoreResult:
-        try:
-            dict_content = json.loads(content)
-            score = float(dict_content["score"])
-            return score_result.ScoreResult(
-                name=self.name,
-                value=score,
-                reason=str(dict_content["reason"]),
-            )
-        except Exception:
-            raise exceptions.MetricComputationError(
-                logging_messages.HALLUCINATION_DETECTION_FAILED
-            )
+        return parser.parse_model_output(content=model_output, name=self.name)

opik/evaluation/metrics/llm_judges/hallucination/parser.py

@@ -0,0 +1,29 @@
+import logging
+import opik.exceptions as exceptions
+import opik.logging_messages as logging_messages
+from opik.evaluation.metrics import score_result
+from opik.evaluation.metrics.llm_judges import parsing_helpers
+
+LOGGER = logging.getLogger(__name__)
+
+
+def parse_model_output(content: str, name: str) -> score_result.ScoreResult:
+    try:
+        dict_content = parsing_helpers.extract_json_content_or_raise(content)
+        score = float(dict_content["score"])
+
+        if not (0.0 <= score <= 1.0):
+            raise exceptions.MetricComputationError(
+                f"Hallucination score must be between 0.0 and 1.0, got {score}"
+            )
+
+        return score_result.ScoreResult(
+            name=name,
+            value=score,
+            reason=str(dict_content["reason"]),
+        )
+    except Exception as e:
+        LOGGER.error(f"Failed to parse model output: {e}", exc_info=True)
+        raise exceptions.MetricComputationError(
+            logging_messages.HALLUCINATION_DETECTION_FAILED
+        )

opik/evaluation/metrics/llm_judges/hallucination/template.py

@@ -88,13 +88,11 @@ def generate_query(
     else:
         examples_str = "\n\nEXAMPLES:\n\n".join(
             [
-                f"<example>\n"
-                f"Input: {example['input']}\n"
-                f"Context: {example['context']}\n"
+                f"<example>\nInput: {example['input']}\nContext: {example['context']}\n"
                 if context is not None
                 else ""
                 f"Output: {example['output']}\n\n"
-                f"{{\"score\": \"{example['score']}\", \"reason\": \"{example['reason']}\"}}\n"
+                f'{{"score": "{example["score"]}", "reason": "{example["reason"]}"}}\n'
                 f"</example>"
                 for i, example in enumerate(few_shot_examples)
             ]

opik/evaluation/metrics/llm_judges/llm_juries/__init__.py

@@ -0,0 +1,3 @@
+from .metric import LLMJuriesJudge
+
+__all__ = ["LLMJuriesJudge"]

opik/evaluation/metrics/llm_judges/llm_juries/metric.py

@@ -0,0 +1,76 @@
+"""LLM Juries: aggregate heterogeneous judges into a consensus score."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Iterable, List, Optional
+
+from opik.evaluation.metrics.base_metric import BaseMetric
+from opik.evaluation.metrics.score_result import ScoreResult
+import opik.exceptions as exceptions
+
+
+class LLMJuriesJudge(BaseMetric):
+    """
+    Aggregate multiple judge metrics into a consensus score.
+
+    Args:
+        judges: Iterable of judge metrics to execute for consensus.
+        name: Display name for the aggregated result. Defaults to
+            ``"llm_juries_judge"``.
+        track: Whether to automatically track results. Defaults to ``True``.
+        project_name: Optional tracking project name. Defaults to ``None``.
+
+    Example:
+        >>> from opik.evaluation.metrics import LLMJuriesJudge, ComplianceRiskJudge
+        >>> juries = LLMJuriesJudge(judges=[ComplianceRiskJudge(model="gpt-4")])
+        >>> result = juries.score(output="Financial guarantees provided.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.6
+    """
+
+    def __init__(
+        self,
+        judges: Iterable[BaseMetric],
+        name: str = "llm_juries_judge",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+        self._judges = list(judges)
+        if not self._judges:
+            raise ValueError("LLMJuriesJudge requires at least one judge metric.")
+
+    def score(self, *args: Any, **kwargs: Any) -> ScoreResult:
+        precomputed: Optional[Dict[BaseMetric, ScoreResult]] = kwargs.pop(
+            "precomputed", None
+        )
+        scores: List[ScoreResult] = []
+        for judge in self._judges:
+            if precomputed is not None and judge in precomputed:
+                raw_result: Any = precomputed[judge]
+            else:
+                raw_result = judge.score(*args, **kwargs)
+            judge_results = raw_result if isinstance(raw_result, list) else [raw_result]
+
+            for result in judge_results:
+                if not isinstance(result, ScoreResult):
+                    raise exceptions.MetricComputationError(
+                        f"Judge {judge.name} returned unexpected result type {type(result)!r}"
+                    )
+                if result.value < 0 or result.value > 1:
+                    raise exceptions.MetricComputationError(
+                        f"Judge {judge.name} returned out-of-range score {result.value}"
+                    )
+                scores.append(result)
+
+        if not scores:
+            raise exceptions.MetricComputationError("No judge scores produced")
+
+        average = sum(res.value for res in scores) / len(scores)
+        metadata = {
+            "judge_scores": {res.name: res.value for res in scores},
+        }
+        reason = f"Averaged {len(scores)} judge scores"
+        return ScoreResult(
+            value=average, name=self.name, reason=reason, metadata=metadata
+        )

opik/evaluation/metrics/llm_judges/moderation/metric.py

@@ -1,14 +1,9 @@
-import json
-import logging
 from typing import Any, List, Optional, Union
+from opik.evaluation.metrics.llm_judges.moderation import parser
 import pydantic
-from opik import logging_messages
 from opik.evaluation.metrics import base_metric, score_result
 from opik.evaluation.models import base_model, models_factory
 from . import template
-from opik import exceptions
-
-LOGGER = logging.getLogger(__name__)
 
 
 class ModerationResponseFormat(pydantic.BaseModel):
@@ -29,6 +24,10 @@ class Moderation(base_metric.BaseMetric):
         name: The name of the metric. Defaults to "moderation_metric".
         few_shot_examples: A list of few-shot examples to be used in the query. If None, default examples will be used.
         track: Whether to track the metric. Defaults to True.
+        project_name: Optional project name to track the metric in for the cases when
+            there are no parent span/trace to inherit project name from.
+        seed: Optional seed value for reproducible model generation. If provided, this seed will be passed to the model for deterministic outputs.
+        temperature: Optional temperature value for model generation. If provided, this temperature will be passed to the model. If not provided, the model's default temperature will be used.
 
     Example:
         >>> from opik.evaluation.metrics import Moderation
@@ -44,22 +43,34 @@ class Moderation(base_metric.BaseMetric):
         name: str = "moderation_metric",
         few_shot_examples: Optional[List[template.FewShotExampleModeration]] = None,
         track: bool = True,
+        project_name: Optional[str] = None,
+        seed: Optional[int] = None,
+        temperature: Optional[float] = None,
     ):
         super().__init__(
             name=name,
             track=track,
+            project_name=project_name,
         )
-
-        self._init_model(model)
+        self._seed = seed
+        self._init_model(model, temperature=temperature)
         self.few_shot_examples = [] if few_shot_examples is None else few_shot_examples
 
     def _init_model(
-        self, model: Optional[Union[str, base_model.OpikBaseModel]]
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]],
+        temperature: Optional[float],
     ) -> None:
         if isinstance(model, base_model.OpikBaseModel):
             self._model = model
         else:
-            self._model = models_factory.get(model_name=model)
+            model_kwargs = {}
+            if temperature is not None:
+                model_kwargs["temperature"] = temperature
+            if self._seed is not None:
+                model_kwargs["seed"] = self._seed
+
+            self._model = models_factory.get(model_name=model, **model_kwargs)
 
     def score(self, output: str, **ignored_kwargs: Any) -> score_result.ScoreResult:
         """
@@ -80,7 +91,7 @@ class Moderation(base_metric.BaseMetric):
             input=llm_query, response_format=ModerationResponseFormat
         )
 
-        return self._parse_model_output(model_output)
+        return parser.parse_model_output(content=model_output, name=self.name)
 
     async def ascore(
         self, output: str, **ignored_kwargs: Any
@@ -106,20 +117,4 @@ class Moderation(base_metric.BaseMetric):
             input=llm_query, response_format=ModerationResponseFormat
         )
 
-        return self._parse_model_output(model_output)
-
-    def _parse_model_output(self, content: str) -> score_result.ScoreResult:
-        try:
-            dict_content = json.loads(content)
-            score: float = float(dict_content["score"])
-
-            if not (0.0 <= score <= 1.0):
-                score = 0.5
-
-            return score_result.ScoreResult(
-                name=self.name, value=score, reason=dict_content["reason"]
-            )
-        except Exception:
-            raise exceptions.MetricComputationError(
-                logging_messages.MODERATION_SCORE_CALC_FAILED
-            )
+        return parser.parse_model_output(content=model_output, name=self.name)

opik/evaluation/metrics/llm_judges/moderation/parser.py

@@ -0,0 +1,27 @@
+import logging
+import opik.exceptions as exceptions
+import opik.logging_messages as logging_messages
+from opik.evaluation.metrics import score_result
+from opik.evaluation.metrics.llm_judges import parsing_helpers
+
+LOGGER = logging.getLogger(__name__)
+
+
+def parse_model_output(content: str, name: str) -> score_result.ScoreResult:
+    try:
+        dict_content = parsing_helpers.extract_json_content_or_raise(content)
+        score: float = float(dict_content["score"])
+
+        if not (0.0 <= score <= 1.0):
+            raise exceptions.MetricComputationError(
+                f"Moderation score must be between 0.0 and 1.0, got {score}"
+            )
+
+        return score_result.ScoreResult(
+            name=name, value=score, reason=dict_content["reason"]
+        )
+    except Exception as e:
+        LOGGER.error(f"Failed to parse model output: {e}", exc_info=True)
+        raise exceptions.MetricComputationError(
+            logging_messages.MODERATION_SCORE_CALC_FAILED
+        )

opik/evaluation/metrics/llm_judges/moderation/template.py

@@ -15,9 +15,9 @@ def generate_query(
     if few_shot_examples:
         examples_str = "\n\n".join(
             [
-                f"Example {i+1}:\n"
+                f"Example {i + 1}:\n"
                 f"Output: {example['output']}\n"
-                f"{{\"score\": {example['score']}, \"reason\": \"{example['reason']}\"}}"
+                f'{{"score": {example["score"]}, "reason": "{example["reason"]}"}}'
                 for i, example in enumerate(few_shot_examples)
             ]
         )

opik/evaluation/metrics/llm_judges/parsing_helpers.py

@@ -0,0 +1,26 @@
+from typing import Any
+import json
+import opik.exceptions as exceptions
+
+
+def extract_json_content_or_raise(content: str) -> Any:
+    try:
+        return json.loads(content)
+    except json.decoder.JSONDecodeError:
+        return _extract_presumably_json_dict_or_raise(content)
+    except Exception as e:
+        raise exceptions.JSONParsingError(
+            f"Failed to parse response to JSON dictionary: {str(e)}"
+        )
+
+
+def _extract_presumably_json_dict_or_raise(content: str) -> str:
+    try:
+        first_paren = content.find("{")
+        last_paren = content.rfind("}")
+        json_string = content[first_paren : last_paren + 1]
+        return json.loads(json_string)
+    except Exception as e:
+        raise exceptions.JSONParsingError(
+            f"Failed to extract presumably JSON dictionary: {str(e)}"
+        )

opik/evaluation/metrics/llm_judges/structure_output_compliance/metric.py

@@ -0,0 +1,144 @@
+from typing import Union, Optional, List, Any
+import logging
+
+from opik.evaluation.models import base_model, models_factory
+from opik.evaluation.metrics import score_result, base_metric
+from opik import exceptions
+from . import template, parser
+from .schema import (
+    FewShotExampleStructuredOutputCompliance,
+    StructuredOutputComplianceResponseFormat,
+)
+
+LOGGER = logging.getLogger(__name__)
+
+
+class StructuredOutputCompliance(base_metric.BaseMetric):
+    """
+    Metric to evaluate whether an LLM's output complies with a specified structured format.
+    This includes checking for valid JSON, JSON-LD compatibility, or adherence to a provided
+    Pydantic/JSON schema.
+
+    Score Range:
+    - Minimum score: 0.0 (complete non-compliance)
+    - Maximum score: 1.0 (complete compliance)
+
+    Score Meaning:
+    - 0.0: Output does not comply with the expected structure at all (e.g., invalid JSON, missing required fields)
+    - 0.5: Partial compliance (e.g., valid JSON but missing some required fields)
+    - 1.0: Complete compliance with the expected structure (valid JSON and all required fields present)
+
+    Args:
+        model: LLM to use for evaluation. Can be a string or an OpikBaseModel instance.
+        name: Metric name.
+        few_shot_examples: Optional few-shot examples to guide the LLM's judgment.
+        track: Whether to track metric execution for observability.
+        project_name: Optional name for tracking in an observability tool.
+        seed: Optional seed value for reproducible model generation. If provided, this seed will be passed to the model for deterministic outputs.
+        temperature: Optional temperature value for model generation. If provided, this temperature will be passed to the model. If not provided, the model's default temperature will be used.
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        name: str = "structured_output_compliance",
+        few_shot_examples: Optional[
+            List[FewShotExampleStructuredOutputCompliance]
+        ] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        seed: Optional[int] = None,
+        temperature: Optional[float] = None,
+    ):
+        super().__init__(name=name, track=track, project_name=project_name)
+        self._seed = seed
+        self._init_model(model, temperature=temperature)
+        self.few_shot_examples = few_shot_examples
+
+    def _init_model(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]],
+        temperature: Optional[float],
+    ) -> None:
+        if isinstance(model, base_model.OpikBaseModel):
+            self._model = model
+        else:
+            model_kwargs = {}
+            if temperature is not None:
+                model_kwargs["temperature"] = temperature
+            if self._seed is not None:
+                model_kwargs["seed"] = self._seed
+
+            self._model = models_factory.get(model_name=model, **model_kwargs)
+
+    def score(
+        self,
+        output: str,
+        schema: Optional[str] = None,
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        """
+        Synchronously compute the structured output compliance score.
+        Args:
+            output: The LLM's output to validate.
+            schema: Optional JSON or Pydantic schema to validate against.
+        Returns:
+            score_result.ScoreResult: An object containing the compliance score and reasons.
+        """
+        try:
+            llm_query = template.generate_query(
+                output=output,
+                schema=schema,
+                few_shot_examples=self.few_shot_examples,
+            )
+
+            model_output = self._model.generate_string(
+                input=llm_query,
+                response_format=StructuredOutputComplianceResponseFormat,
+            )
+
+            return parser.parse_model_output(content=model_output, name=self.name)
+
+        except Exception as e:
+            LOGGER.error(
+                f"Structured output compliance evaluation failed: {e}", exc_info=True
+            )
+            raise exceptions.MetricComputationError(
+                f"Structured output compliance evaluation failed: {str(e)}"
+            ) from e
+
+    async def ascore(
+        self,
+        output: str,
+        schema: Optional[str] = None,
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        """
+        Asynchronously compute the structured output compliance score.
+        Args:
+            output: The LLM's output to validate.
+            schema: Optional JSON or Pydantic schema to validate against.
+        Returns:
+            score_result.ScoreResult: An object containing the compliance score and reasons.
+        """
+        try:
+            llm_query = template.generate_query(
+                output=output,
+                schema=schema,
+                few_shot_examples=self.few_shot_examples,
+            )
+
+            model_output = await self._model.agenerate_string(
+                input=llm_query,
+                response_format=StructuredOutputComplianceResponseFormat,
+            )
+
+            return parser.parse_model_output(content=model_output, name=self.name)
+
+        except Exception as e:
+            LOGGER.error(
+                f"Structured output compliance evaluation failed: {e}", exc_info=True
+            )
+            raise exceptions.MetricComputationError(
+                f"Structured output compliance evaluation failed: {str(e)}"
+            ) from e