opik 1.8.39__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/metrics/llm_judges/g_eval_presets/bias_classifier.py

@@ -0,0 +1,181 @@
+from __future__ import annotations
+
+from typing import Optional, Union
+
+from opik.evaluation.metrics.llm_judges.g_eval import metric as g_eval_metric
+from opik.evaluation.models import base_model
+
+
+class DemographicBiasJudge(g_eval_metric.GEvalPreset):
+    """
+    Score demographic stereotyping or bias in a response.
+
+    Args:
+        model: Optional model identifier or ``OpikBaseModel`` instance.
+        track: Whether to automatically track results. Defaults to ``True``.
+        project_name: Optional tracking project name.
+        temperature: Sampling temperature forwarded to the model.
+
+    Example:
+        >>> from opik.evaluation.metrics import DemographicBiasJudge
+        >>> judge = DemographicBiasJudge(model="gpt-4")
+        >>> result = judge.score(output="People from X group are always late.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.95
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        temperature: float = 0.0,
+    ) -> None:
+        super().__init__(
+            preset="bias_demographic",
+            model=model,
+            track=track,
+            project_name=project_name,
+            temperature=temperature,
+            name="demographic_bias_judge",
+        )
+
+
+class PoliticalBiasJudge(g_eval_metric.GEvalPreset):
+    """
+    Detect partisan or ideological bias in a response.
+
+    Args:
+        model: Optional model identifier or ``OpikBaseModel`` instance.
+        track: Whether to automatically track results. Defaults to ``True``.
+        project_name: Optional tracking project name.
+        temperature: Sampling temperature forwarded to the model.
+
+    Example:
+        >>> from opik.evaluation.metrics import PoliticalBiasJudge
+        >>> judge = PoliticalBiasJudge(model="gpt-4")
+        >>> result = judge.score(output="Vote for candidate X because Y is corrupt")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.87
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        temperature: float = 0.0,
+    ) -> None:
+        super().__init__(
+            preset="bias_political",
+            model=model,
+            track=track,
+            project_name=project_name,
+            temperature=temperature,
+            name="political_bias_judge",
+        )
+
+
+class GenderBiasJudge(g_eval_metric.GEvalPreset):
+    """
+    Detect gender stereotyping or exclusion in generated text.
+
+    Args:
+        model: Optional model identifier or ``OpikBaseModel`` instance.
+        track: Whether to automatically track results. Defaults to ``True``.
+        project_name: Optional tracking project name.
+        temperature: Sampling temperature forwarded to the model.
+
+    Example:
+        >>> from opik.evaluation.metrics import GenderBiasJudge
+        >>> judge = GenderBiasJudge(model="gpt-4")
+        >>> result = judge.score(output="Women are naturally worse at math.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.93
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        temperature: float = 0.0,
+    ) -> None:
+        super().__init__(
+            preset="bias_gender",
+            model=model,
+            track=track,
+            project_name=project_name,
+            temperature=temperature,
+            name="gender_bias_judge",
+        )
+
+
+class ReligiousBiasJudge(g_eval_metric.GEvalPreset):
+    """
+    Evaluate responses for religious bias or disrespectful language.
+
+    Args:
+        model: Optional model identifier or ``OpikBaseModel`` instance.
+        track: Whether to automatically track results. Defaults to ``True``.
+        project_name: Optional tracking project name.
+        temperature: Sampling temperature forwarded to the model.
+
+    Example:
+        >>> from opik.evaluation.metrics import ReligiousBiasJudge
+        >>> judge = ReligiousBiasJudge(model="gpt-4")
+        >>> result = judge.score(output="Believers of X are all foolish.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.9
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        temperature: float = 0.0,
+    ) -> None:
+        super().__init__(
+            preset="bias_religion",
+            model=model,
+            track=track,
+            project_name=project_name,
+            temperature=temperature,
+            name="religious_bias_judge",
+        )
+
+
+class RegionalBiasJudge(g_eval_metric.GEvalPreset):
+    """
+    Assess geographic or cultural bias in responses.
+
+    Args:
+        model: Optional model identifier or ``OpikBaseModel`` instance.
+        track: Whether to automatically track results. Defaults to ``True``.
+        project_name: Optional tracking project name.
+        temperature: Sampling temperature forwarded to the model.
+
+    Example:
+        >>> from opik.evaluation.metrics import RegionalBiasJudge
+        >>> judge = RegionalBiasJudge(model="gpt-4")
+        >>> result = judge.score(output="People from region Z are lazy.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.88
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        temperature: float = 0.0,
+    ) -> None:
+        super().__init__(
+            preset="bias_regional",
+            model=model,
+            track=track,
+            project_name=project_name,
+            temperature=temperature,
+            name="regional_bias_judge",
+        )

opik/evaluation/metrics/llm_judges/g_eval_presets/compliance_risk.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from typing import Optional, Union
+
+from opik.evaluation.metrics.llm_judges.g_eval import metric as g_eval_metric
+from opik.evaluation.models import base_model
+
+
+class ComplianceRiskJudge(g_eval_metric.GEvalPreset):
+    """
+    Evaluate responses for non-compliant or misleading claims in regulated sectors.
+
+    Args:
+        model: Optional model identifier or ``OpikBaseModel`` instance.
+        track: Whether to automatically track judge outputs. Defaults to ``True``.
+        project_name: Optional tracking project name.
+        temperature: Sampling temperature forwarded to the underlying model.
+
+    Example:
+        >>> from opik.evaluation.metrics import ComplianceRiskJudge
+        >>> judge = ComplianceRiskJudge(model="gpt-4")
+        >>> result = judge.score(output="This pill cures diabetes in a week.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.97
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        temperature: float = 0.0,
+    ) -> None:
+        super().__init__(
+            preset="compliance_regulated_truthfulness",
+            model=model,
+            track=track,
+            project_name=project_name,
+            temperature=temperature,
+            name="compliance_risk_judge",
+        )

opik/evaluation/metrics/llm_judges/g_eval_presets/prompt_uncertainty.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from typing import Optional, Union
+
+from opik.evaluation.metrics.llm_judges.g_eval import metric as g_eval_metric
+from opik.evaluation.models import base_model
+
+
+class PromptUncertaintyJudge(g_eval_metric.GEvalPreset):
+    """
+    Rate how ambiguous or underspecified a prompt feels to the model.
+
+    Args:
+        model: Optional model identifier or ``OpikBaseModel`` instance.
+        track: Whether to automatically track judge outputs. Defaults to ``True``.
+        project_name: Optional tracking project name.
+        temperature: Sampling temperature forwarded to the underlying model.
+
+    Example:
+        >>> from opik.evaluation.metrics import PromptUncertaintyJudge
+        >>> judge = PromptUncertaintyJudge(model="gpt-4")
+        >>> result = judge.score(output="Do the right thing in the best way possible.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.8
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        temperature: float = 0.0,
+    ) -> None:
+        super().__init__(
+            preset="prompt_uncertainty",
+            model=model,
+            track=track,
+            project_name=project_name,
+            temperature=temperature,
+            name="prompt_uncertainty_judge",
+        )

opik/evaluation/metrics/llm_judges/g_eval_presets/qa_suite.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from typing import Optional, Union
+
+from opik.evaluation.metrics.llm_judges.g_eval import metric as g_eval_metric
+from opik.evaluation.models import base_model
+
+
+class SummarizationConsistencyJudge(g_eval_metric.GEvalPreset):
+    """
+    Score how faithful a summary is to its source content.
+
+    Args:
+        model: Optional model identifier or ``OpikBaseModel`` instance.
+        track: Whether to automatically track judge outputs. Defaults to ``True``.
+        project_name: Optional tracking project name.
+        temperature: Sampling temperature forwarded to the underlying model.
+
+    Example:
+        >>> from opik.evaluation.metrics import SummarizationConsistencyJudge
+        >>> judge = SummarizationConsistencyJudge(model="gpt-4")
+        >>> result = judge.score(output="Summary omits key fact.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.4
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        temperature: float = 0.0,
+    ) -> None:
+        super().__init__(
+            preset="summarization_consistency",
+            model=model,
+            track=track,
+            project_name=project_name,
+            temperature=temperature,
+            name="summarization_consistency_judge",
+        )
+
+
+class SummarizationCoherenceJudge(g_eval_metric.GEvalPreset):
+    """
+    Evaluate the coherence and structure of generated summaries.
+
+    Args:
+        model: Optional model identifier or ``OpikBaseModel`` instance.
+        track: Whether to automatically track judge outputs. Defaults to ``True``.
+        project_name: Optional tracking project name.
+        temperature: Sampling temperature forwarded to the underlying model.
+
+    Example:
+        >>> from opik.evaluation.metrics import SummarizationCoherenceJudge
+        >>> judge = SummarizationCoherenceJudge(model="gpt-4")
+        >>> result = judge.score(output="Summary jumps between unrelated topics.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.5
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        temperature: float = 0.0,
+    ) -> None:
+        super().__init__(
+            preset="summarization_coherence",
+            model=model,
+            track=track,
+            project_name=project_name,
+            temperature=temperature,
+            name="summarization_coherence_judge",
+        )
+
+
+class DialogueHelpfulnessJudge(g_eval_metric.GEvalPreset):
+    """
+    Judge how helpful an assistant reply is within a dialogue.
+
+    Args:
+        model: Optional model identifier or ``OpikBaseModel`` instance.
+        track: Whether to automatically track judge outputs. Defaults to ``True``.
+        project_name: Optional tracking project name.
+        temperature: Sampling temperature forwarded to the underlying model.
+
+    Example:
+        >>> from opik.evaluation.metrics import DialogueHelpfulnessJudge
+        >>> judge = DialogueHelpfulnessJudge(model="gpt-4")
+        >>> result = judge.score(output="Assistant politely refuses without help.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.3
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        temperature: float = 0.0,
+    ) -> None:
+        super().__init__(
+            preset="dialogue_helpfulness",
+            model=model,
+            track=track,
+            project_name=project_name,
+            temperature=temperature,
+            name="dialogue_helpfulness_judge",
+        )
+
+
+class QARelevanceJudge(g_eval_metric.GEvalPreset):
+    """
+    Check whether an answer directly addresses the user question.
+
+    Args:
+        model: Optional model identifier or ``OpikBaseModel`` instance.
+        track: Whether to automatically track judge outputs. Defaults to ``True``.
+        project_name: Optional tracking project name.
+        temperature: Sampling temperature forwarded to the underlying model.
+
+    Example:
+        >>> from opik.evaluation.metrics import QARelevanceJudge
+        >>> judge = QARelevanceJudge(model="gpt-4")
+        >>> result = judge.score(output="Answer rambles without addressing the ask.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.2
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        temperature: float = 0.0,
+    ) -> None:
+        super().__init__(
+            preset="qa_relevance",
+            model=model,
+            track=track,
+            project_name=project_name,
+            temperature=temperature,
+            name="qa_relevance_judge",
+        )

opik/evaluation/metrics/llm_judges/hallucination/metric.py

@@ -27,6 +27,8 @@ class Hallucination(base_metric.BaseMetric):
         track: Whether to track the metric. Defaults to True.
         project_name: Optional project name to track the metric in for the cases when
             there are no parent span/trace to inherit project name from.
+        seed: Optional seed value for reproducible model generation. If provided, this seed will be passed to the model for deterministic outputs.
+        temperature: Optional temperature value for model generation. If provided, this temperature will be passed to the model. If not provided, the model's default temperature will be used.
 
     Example:
         >>> from opik.evaluation.metrics import Hallucination
@@ -49,18 +51,29 @@ class Hallucination(base_metric.BaseMetric):
         few_shot_examples: Optional[List[template.FewShotExampleHallucination]] = None,
         track: bool = True,
         project_name: Optional[str] = None,
+        seed: Optional[int] = None,
+        temperature: Optional[float] = None,
     ):
         super().__init__(name=name, track=track, project_name=project_name)
-        self._init_model(model)
+        self._seed = seed
+        self._init_model(model, temperature=temperature)
         self.few_shot_examples = few_shot_examples
 
     def _init_model(
-        self, model: Optional[Union[str, base_model.OpikBaseModel]]
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]],
+        temperature: Optional[float],
     ) -> None:
         if isinstance(model, base_model.OpikBaseModel):
             self._model = model
         else:
-            self._model = models_factory.get(model_name=model)
+            model_kwargs = {}
+            if temperature is not None:
+                model_kwargs["temperature"] = temperature
+            if self._seed is not None:
+                model_kwargs["seed"] = self._seed
+
+            self._model = models_factory.get(model_name=model, **model_kwargs)
 
     def score(
         self,

opik/evaluation/metrics/llm_judges/llm_juries/__init__.py

@@ -0,0 +1,3 @@
+from .metric import LLMJuriesJudge
+
+__all__ = ["LLMJuriesJudge"]

opik/evaluation/metrics/llm_judges/llm_juries/metric.py

@@ -0,0 +1,76 @@
+"""LLM Juries: aggregate heterogeneous judges into a consensus score."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Iterable, List, Optional
+
+from opik.evaluation.metrics.base_metric import BaseMetric
+from opik.evaluation.metrics.score_result import ScoreResult
+import opik.exceptions as exceptions
+
+
+class LLMJuriesJudge(BaseMetric):
+    """
+    Aggregate multiple judge metrics into a consensus score.
+
+    Args:
+        judges: Iterable of judge metrics to execute for consensus.
+        name: Display name for the aggregated result. Defaults to
+            ``"llm_juries_judge"``.
+        track: Whether to automatically track results. Defaults to ``True``.
+        project_name: Optional tracking project name. Defaults to ``None``.
+
+    Example:
+        >>> from opik.evaluation.metrics import LLMJuriesJudge, ComplianceRiskJudge
+        >>> juries = LLMJuriesJudge(judges=[ComplianceRiskJudge(model="gpt-4")])
+        >>> result = juries.score(output="Financial guarantees provided.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        0.6
+    """
+
+    def __init__(
+        self,
+        judges: Iterable[BaseMetric],
+        name: str = "llm_juries_judge",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+        self._judges = list(judges)
+        if not self._judges:
+            raise ValueError("LLMJuriesJudge requires at least one judge metric.")
+
+    def score(self, *args: Any, **kwargs: Any) -> ScoreResult:
+        precomputed: Optional[Dict[BaseMetric, ScoreResult]] = kwargs.pop(
+            "precomputed", None
+        )
+        scores: List[ScoreResult] = []
+        for judge in self._judges:
+            if precomputed is not None and judge in precomputed:
+                raw_result: Any = precomputed[judge]
+            else:
+                raw_result = judge.score(*args, **kwargs)
+            judge_results = raw_result if isinstance(raw_result, list) else [raw_result]
+
+            for result in judge_results:
+                if not isinstance(result, ScoreResult):
+                    raise exceptions.MetricComputationError(
+                        f"Judge {judge.name} returned unexpected result type {type(result)!r}"
+                    )
+                if result.value < 0 or result.value > 1:
+                    raise exceptions.MetricComputationError(
+                        f"Judge {judge.name} returned out-of-range score {result.value}"
+                    )
+                scores.append(result)
+
+        if not scores:
+            raise exceptions.MetricComputationError("No judge scores produced")
+
+        average = sum(res.value for res in scores) / len(scores)
+        metadata = {
+            "judge_scores": {res.name: res.value for res in scores},
+        }
+        reason = f"Averaged {len(scores)} judge scores"
+        return ScoreResult(
+            value=average, name=self.name, reason=reason, metadata=metadata
+        )

opik/evaluation/metrics/llm_judges/moderation/metric.py

@@ -26,6 +26,8 @@ class Moderation(base_metric.BaseMetric):
         track: Whether to track the metric. Defaults to True.
         project_name: Optional project name to track the metric in for the cases when
             there are no parent span/trace to inherit project name from.
+        seed: Optional seed value for reproducible model generation. If provided, this seed will be passed to the model for deterministic outputs.
+        temperature: Optional temperature value for model generation. If provided, this temperature will be passed to the model. If not provided, the model's default temperature will be used.
 
     Example:
         >>> from opik.evaluation.metrics import Moderation
@@ -42,23 +44,33 @@ class Moderation(base_metric.BaseMetric):
         few_shot_examples: Optional[List[template.FewShotExampleModeration]] = None,
         track: bool = True,
         project_name: Optional[str] = None,
+        seed: Optional[int] = None,
+        temperature: Optional[float] = None,
     ):
         super().__init__(
             name=name,
             track=track,
             project_name=project_name,
         )
-
-        self._init_model(model)
+        self._seed = seed
+        self._init_model(model, temperature=temperature)
         self.few_shot_examples = [] if few_shot_examples is None else few_shot_examples
 
     def _init_model(
-        self, model: Optional[Union[str, base_model.OpikBaseModel]]
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]],
+        temperature: Optional[float],
     ) -> None:
         if isinstance(model, base_model.OpikBaseModel):
             self._model = model
         else:
-            self._model = models_factory.get(model_name=model)
+            model_kwargs = {}
+            if temperature is not None:
+                model_kwargs["temperature"] = temperature
+            if self._seed is not None:
+                model_kwargs["seed"] = self._seed
+
+            self._model = models_factory.get(model_name=model, **model_kwargs)
 
     def score(self, output: str, **ignored_kwargs: Any) -> score_result.ScoreResult:
         """