opik 1.6.4__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/metrics/llm_judges/trajectory_accuracy/metric.py

@@ -0,0 +1,171 @@
+from typing import Dict, Any, Optional, List, Union
+import logging
+import pydantic
+
+from opik.evaluation.models import base_model, models_factory
+from opik.evaluation.metrics import base_metric, score_result
+from opik.evaluation.metrics.llm_judges.trajectory_accuracy import templates, parser
+from opik import exceptions
+
+LOGGER = logging.getLogger(__name__)
+
+
+class TrajectoryAccuracyResponseFormat(pydantic.BaseModel):
+    """Expected format for LLM response when evaluating trajectory accuracy."""
+
+    score: float = pydantic.Field(
+        ..., ge=0.0, le=1.0, description="Score between 0.0 and 1.0"
+    )
+    explanation: str = pydantic.Field(
+        ..., min_length=1, description="Detailed explanation for the score"
+    )
+
+
+class TrajectoryAccuracy(base_metric.BaseMetric):
+    """
+    A metric that evaluates the accuracy of ReAct-style agent trajectories.
+
+    This metric uses an LLM to judge whether an agent's sequence of thought/action/observation
+    steps demonstrates effective reasoning and appropriate action selection to achieve the goal.
+    It returns a score between 0.0 and 1.0 based on reasoning quality, action appropriateness,
+    observation integration, goal achievement, and efficiency.
+
+    Args:
+        model: The LLM to use for evaluation. Can be a string (model name) or an
+            `opik.evaluation.models.OpikBaseModel` subclass instance.
+            `opik.evaluation.models.LiteLLMChatModel` is used by default.
+        name: The name of the metric.
+        track: Whether to track the metric. Defaults to True.
+        project_name: Optional project name to track the metric in for the cases when
+            there are no parent span/trace to inherit project name from.
+        seed: Optional seed value for reproducible model generation. If provided, this seed will be passed to the model for deterministic outputs.
+        temperature: Optional temperature value for model generation. If provided, this temperature will be passed to the model. If not provided, the model's default temperature will be used.
+
+    Example:
+        >>> from opik.evaluation.metrics import TrajectoryAccuracy
+        >>> trajectory_metric = TrajectoryAccuracy()
+        >>> result = trajectory_metric.score(
+        ...     goal="Find the weather in Paris",
+        ...     trajectory=[{
+        ...         'thought': 'I need to search for weather information',
+        ...         'action': 'search_weather(location="Paris")',
+        ...         'observation': 'Weather: 22°C, sunny'
+        ...     }],
+        ...     final_result="The weather in Paris is 22°C and sunny"
+        ... )
+        >>> print(result.value)  # A float between 0.0 and 1.0
+        >>> print(result.reason)  # Explanation for the score
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        name: str = "trajectory_accuracy_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+        seed: Optional[int] = None,
+        temperature: Optional[float] = None,
+    ):
+        super().__init__(name=name, track=track, project_name=project_name)
+        self._seed = seed
+        self._init_model(model, temperature=temperature)
+
+    def _init_model(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]],
+        temperature: Optional[float],
+    ) -> None:
+        if isinstance(model, base_model.OpikBaseModel):
+            self._model = model
+        else:
+            model_kwargs = {}
+            if temperature is not None:
+                model_kwargs["temperature"] = temperature
+            if self._seed is not None:
+                model_kwargs["seed"] = self._seed
+
+            self._model = models_factory.get(model_name=model, **model_kwargs)
+
+    def score(
+        self,
+        goal: str,
+        trajectory: List[Dict[str, Any]],
+        final_result: str,
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        """
+        Calculate the trajectory accuracy score for the given ReAct-style agent trajectory.
+
+        Args:
+            goal: The intended goal or task description.
+            trajectory: List of steps with 'thought', 'action', 'observation' keys.
+            final_result: The final outcome achieved.
+            **ignored_kwargs: Additional keyword arguments that are ignored.
+
+        Returns:
+            score_result.ScoreResult: A ScoreResult object with a value between 0.0 and 1.0
+                indicating trajectory accuracy, along with an explanation for the verdict.
+        """
+        try:
+            example = {
+                "goal": goal,
+                "trajectory": trajectory,
+                "final_result": final_result,
+            }
+
+            prompt = templates.create_evaluation_prompt(example)
+
+            response = self._model.generate_string(
+                input=prompt,
+                response_format=TrajectoryAccuracyResponseFormat,
+            )
+
+            return parser.parse_evaluation_response(response, self.name)
+
+        except Exception as e:
+            LOGGER.error(f"Trajectory accuracy evaluation failed: {e}", exc_info=True)
+            raise exceptions.MetricComputationError(
+                f"Trajectory accuracy evaluation failed: {str(e)}"
+            ) from e
+
+    async def ascore(
+        self,
+        goal: str,
+        trajectory: List[Dict[str, Any]],
+        final_result: str,
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        """
+        Asynchronously calculate the trajectory accuracy score.
+
+        Args:
+            goal: The intended goal or task description.
+            trajectory: List of steps with 'thought', 'action', 'observation' keys.
+            final_result: The final outcome achieved.
+            **ignored_kwargs: Additional keyword arguments that are ignored.
+
+        Returns:
+            score_result.ScoreResult: A ScoreResult object with a value between 0.0 and 1.0
+                indicating trajectory accuracy, along with an explanation for the verdict.
+        """
+        try:
+            example = {
+                "goal": goal,
+                "trajectory": trajectory,
+                "final_result": final_result,
+            }
+
+            prompt = templates.create_evaluation_prompt(example)
+
+            response = await self._model.agenerate_string(
+                input=prompt,
+                response_format=TrajectoryAccuracyResponseFormat,
+            )
+
+            return parser.parse_evaluation_response(response, self.name)
+
+        except Exception as e:
+            LOGGER.error(f"Trajectory accuracy evaluation failed: {e}", exc_info=True)
+            raise exceptions.MetricComputationError(
+                f"Trajectory accuracy evaluation failed: {str(e)}"
+            ) from e

opik/evaluation/metrics/llm_judges/trajectory_accuracy/parser.py

@@ -0,0 +1,38 @@
+from opik import exceptions
+from opik.evaluation.metrics import score_result
+from opik.evaluation.metrics.llm_judges import parsing_helpers
+
+
+def parse_evaluation_response(
+    content: str, metric_name: str
+) -> score_result.ScoreResult:
+    """Parse LLM response and extract score and explanation."""
+    try:
+        parsed_content = parsing_helpers.extract_json_content_or_raise(content)
+
+        score = float(parsed_content["score"])
+        explanation = str(parsed_content["explanation"])
+
+        if not (0.0 <= score <= 1.0):
+            raise ValueError(f"Score {score} outside valid range [0.0, 1.0]")
+
+        if not explanation.strip():
+            raise ValueError("Explanation cannot be empty")
+
+        return score_result.ScoreResult(
+            name=metric_name, value=score, reason=explanation
+        )
+
+    except KeyError as e:
+        missing_key = str(e).strip("'\"")
+        raise exceptions.MetricComputationError(
+            f"Missing required field in response: {missing_key}"
+        )
+    except (ValueError, TypeError) as e:
+        raise exceptions.MetricComputationError(f"Invalid response format: {str(e)}")
+    except exceptions.JSONParsingError:
+        raise
+    except Exception as e:
+        raise exceptions.MetricComputationError(
+            f"Failed to parse trajectory accuracy evaluation: {str(e)}"
+        )

opik/evaluation/metrics/llm_judges/trajectory_accuracy/templates.py

@@ -0,0 +1,65 @@
+from typing import Dict, Any, List
+
+EVALUATION_PROMPT_TEMPLATE = """You are an expert evaluator of ReAct-style agent trajectories. Assess how effectively the agent reasoned through the problem and selected appropriate actions.
+
+Evaluation Criteria:
+1. Reasoning Quality: Logical, relevant thoughts that guide action selection
+2. Action Appropriateness: Actions align with thoughts and progress toward the goal
+3. Observation Integration: Effective use of feedback to inform next steps
+4. Goal Achievement: Successfully accomplishes the stated objective
+5. Efficiency: Reasonable path without unnecessary detours
+
+Scoring Guidelines:
+- 0.9-1.0: Excellent reasoning, appropriate actions, achieves goal efficiently
+- 0.7-0.8: Good performance with minor issues or inefficiencies
+- 0.5-0.6: Adequate but with notable problems in reasoning or actions
+- 0.3-0.4: Poor performance, significant flaws but some progress
+- 0.0-0.2: Fundamentally flawed, fails to achieve goal
+
+GOAL: {goal}
+
+TRAJECTORY:
+{trajectory_steps}
+
+FINAL RESULT: {final_result}
+
+Respond in JSON format:
+{{
+    "score": <float between 0.0 and 1.0>,
+    "explanation": "<specific evaluation referencing trajectory steps>"
+}}"""
+
+
+def create_evaluation_prompt(example: Dict[str, Any]) -> str:
+    """Create the evaluation prompt for trajectory assessment."""
+
+    goal = example.get("goal", "No goal specified")
+    trajectory = example.get("trajectory", [])
+    final_result = example.get("final_result", "No result specified")
+
+    trajectory_steps = _format_trajectory_steps(trajectory)
+
+    return EVALUATION_PROMPT_TEMPLATE.format(
+        goal=goal, trajectory_steps=trajectory_steps, final_result=final_result
+    )
+
+
+def _format_trajectory_steps(trajectory: List[Dict[str, Any]]) -> str:
+    """Format trajectory steps for prompt inclusion."""
+    if not trajectory:
+        return "No trajectory steps provided"
+
+    formatted_steps = []
+    for i, step in enumerate(trajectory, 1):
+        thought = step.get("thought", "No thought")
+        action = step.get("action", "No action")
+        observation = step.get("observation", "No observation")
+
+        formatted_steps.append(
+            f"Step {i}:\n"
+            f"  Thought: {thought}\n"
+            f"  Action: {action}\n"
+            f"  Observation: {observation}"
+        )
+
+    return "\n\n".join(formatted_steps)

opik/evaluation/metrics/llm_judges/usefulness/metric.py

@@ -1,14 +1,9 @@
-import json
-import logging
 from typing import Union, Optional, Any
 import pydantic
 from opik.evaluation.models import base_model, models_factory
 from opik.evaluation.metrics import score_result, base_metric
 
-from . import template
-from opik.exceptions import MetricComputationError
-
-LOGGER = logging.getLogger(__name__)
+from . import template, parser
 
 
 class UsefulnessResponseFormat(pydantic.BaseModel):
@@ -29,6 +24,10 @@ class Usefulness(base_metric.BaseMetric):
             `opik.evaluation.models.LiteLLMChatModel` is used by default.
         name: The name of the metric. Defaults to "UsefulnessMetric".
         track: Whether to track the metric. Defaults to True.
+        project_name: Optional project name to track the metric in for the cases when
+            there are no parent span/trace to inherit project name from.
+        seed: Optional seed value for reproducible model generation. If provided, this seed will be passed to the model for deterministic outputs.
+        temperature: Optional temperature value for model generation. If provided, this temperature will be passed to the model. If not provided, the model's default temperature will be used.
 
     Example:
         >>> from opik.evaluation.metrics import Usefulness
@@ -43,21 +42,33 @@ class Usefulness(base_metric.BaseMetric):
         model: Optional[Union[str, base_model.OpikBaseModel]] = None,
         name: str = "UsefulnessMetric",
         track: bool = True,
+        project_name: Optional[str] = None,
+        seed: Optional[int] = None,
+        temperature: Optional[float] = None,
     ):
         super().__init__(
             name=name,
             track=track,
+            project_name=project_name,
         )
-
-        self._init_model(model)
+        self._seed = seed
+        self._init_model(model, temperature=temperature)
 
     def _init_model(
-        self, model: Optional[Union[str, base_model.OpikBaseModel]]
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]],
+        temperature: Optional[float],
     ) -> None:
         if isinstance(model, base_model.OpikBaseModel):
             self._model = model
         else:
-            self._model = models_factory.get(model_name=model)
+            model_kwargs = {}
+            if temperature is not None:
+                model_kwargs["temperature"] = temperature
+            if self._seed is not None:
+                model_kwargs["seed"] = self._seed
+
+            self._model = models_factory.get(model_name=model, **model_kwargs)
 
     def score(
         self, input: str, output: str, **ignored_kwargs: Any
@@ -82,7 +93,7 @@ class Usefulness(base_metric.BaseMetric):
             input=llm_query, response_format=UsefulnessResponseFormat
         )
 
-        return self._parse_model_output(model_output)
+        return parser.parse_model_output(content=model_output, name=self.name)
 
     async def ascore(
         self, input: str, output: str, **ignored_kwargs: Any
@@ -107,24 +118,4 @@ class Usefulness(base_metric.BaseMetric):
             input=llm_query, response_format=UsefulnessResponseFormat
         )
 
-        return self._parse_model_output(model_output)
-
-    def _parse_model_output(self, content: str) -> score_result.ScoreResult:
-        """Parse the model output string into a ScoreResult."""
-        try:
-            dict_content = json.loads(content)
-            score: float = float(dict_content["score"])
-
-            if not (0.0 <= score <= 1.0):
-                raise MetricComputationError(
-                    f"Usefulness score must be between 0.0 and 1.0, got {score}"
-                )
-
-            return score_result.ScoreResult(
-                name=self.name, value=score, reason=dict_content["reason"]
-            )
-        except Exception as e:
-            LOGGER.error(f"Failed to parse model output: {e}", exc_info=True)
-            raise MetricComputationError(
-                f"Failed to parse usefulness score from model output: {repr(e)}"
-            )
+        return parser.parse_model_output(content=model_output, name=self.name)

opik/evaluation/metrics/llm_judges/usefulness/parser.py

@@ -0,0 +1,28 @@
+import logging
+import opik.logging_messages as logging_messages
+import opik.exceptions as exceptions
+from opik.evaluation.metrics import score_result
+from opik.evaluation.metrics.llm_judges import parsing_helpers
+
+LOGGER = logging.getLogger(__name__)
+
+
+def parse_model_output(content: str, name: str) -> score_result.ScoreResult:
+    """Parse the model output string into a ScoreResult."""
+    try:
+        dict_content = parsing_helpers.extract_json_content_or_raise(content)
+        score: float = float(dict_content["score"])
+
+        if not (0.0 <= score <= 1.0):
+            raise exceptions.MetricComputationError(
+                f"Usefulness score must be between 0.0 and 1.0, got {score}"
+            )
+
+        return score_result.ScoreResult(
+            name=name, value=score, reason=dict_content["reason"]
+        )
+    except Exception as e:
+        LOGGER.error(f"Failed to parse model output: {e}", exc_info=True)
+        raise exceptions.MetricComputationError(
+            logging_messages.USEFULNESS_SCORE_CALC_FAILED
+        )

opik/evaluation/metrics/ragas_metric.py

@@ -0,0 +1,112 @@
+from opik.evaluation.metrics import base_metric, score_result
+import opik.exceptions as exceptions
+
+from typing import Dict, Any, Optional, TYPE_CHECKING
+import opik.opik_context as opik_context
+
+if TYPE_CHECKING:
+    from ragas import metrics as ragas_metrics
+    from ragas import dataset_schema as ragas_dataset_schema
+    from opik.integrations.langchain import OpikTracer
+
+
+class RagasMetricWrapper(base_metric.BaseMetric):
+    def __init__(
+        self,
+        ragas_metric: "ragas_metrics.SingleTurnMetric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ):
+        from ragas import metrics as ragas_metrics
+
+        # Provide a better error message if user try to pass a Ragas metric that is not a single turn metric
+        if not isinstance(ragas_metric, ragas_metrics.SingleTurnMetric):
+            raise ValueError(
+                f"ragas_metric {type(ragas_metric)} is not a SingleTurnMetric Ragas Metric"
+            )
+        super().__init__(name=ragas_metric.name, track=track, project_name=project_name)
+        self.ragas_metric = ragas_metric
+        self._required_fields = ragas_metric.required_columns[
+            ragas_metrics.MetricType.SINGLE_TURN.name
+        ]
+
+    def _create_ragas_single_turn_sample(
+        self, input_dict: Dict[str, Any]
+    ) -> "ragas_dataset_schema.SingleTurnSample":
+        from ragas import dataset_schema as ragas_dataset_schema
+
+        # Add basic field name mapping between Opik and Ragas
+        if "user_input" not in input_dict and "input" in input_dict:
+            input_dict["user_input"] = input_dict["input"]
+
+        if "response" not in input_dict and "output" in input_dict:
+            input_dict["response"] = input_dict["output"]
+
+        sample_dict = {}
+        missing_arguments = []
+        for field in self._required_fields:
+            try:
+                sample_dict[field] = input_dict[field]
+            except KeyError:
+                missing_arguments.append(field)
+
+        if len(missing_arguments) > 0:
+            raise exceptions.ScoreMethodMissingArguments(
+                self.name,
+                missing_arguments,
+                list(input_dict.keys()),
+            )
+
+        sample = ragas_dataset_schema.SingleTurnSample(**sample_dict)
+        return sample
+
+    async def ascore(self, **kwargs: Any) -> score_result.ScoreResult:
+        sample = self._create_ragas_single_turn_sample(kwargs)
+
+        callbacks = [_get_opik_tracer_instance()] if self.track else []
+
+        score = await self.ragas_metric.single_turn_ascore(sample, callbacks=callbacks)
+        return score_result.ScoreResult(value=score, name=self.name)
+
+    def score(self, **kwargs: Any) -> score_result.ScoreResult:
+        sample = self._create_ragas_single_turn_sample(kwargs)
+
+        callbacks = [_get_opik_tracer_instance()] if self.track else []
+
+        score = self.ragas_metric.single_turn_score(sample, callbacks=callbacks)
+        return score_result.ScoreResult(value=score, name=self.name)
+
+
+def _get_opik_tracer_instance() -> "OpikTracer":
+    from opik.integrations.langchain import OpikTracer
+
+    current_span_data = opik_context.get_current_span_data()
+    current_trace_data = opik_context.get_current_trace_data()
+    project_name = None
+
+    if current_span_data is not None:
+        project_name = (
+            current_trace_data.project_name
+            if current_trace_data is not None
+            else current_span_data.project_name
+        )
+
+    # OPIK-3505: Why opik_context_read_only_mode=True?
+    #
+    # Problem: Ragas runs metrics concurrently under the hood with a manual management
+    # of the event loop. It was discovered that these metrics share the same context and so
+    # ContextVar used in Opik context storage can't be modified safely by them because concurrent
+    # operations share the same span stack.
+    #
+    # Solution: Disable context modification (opik_context_read_only_mode=True).
+    # OpikTracer will still create spans/traces and track parent-child relationships
+    # using LangChain's Run IDs, but won't modify the shared ContextVar storage.
+    #
+    # Trade-off: @track-decorated functions called within Ragas won't be attached
+    # to the Ragas spans. This is acceptable since Ragas metrics are self-contained
+    # and don't typically call user-defined tracked functions.
+    opik_tracer = OpikTracer(
+        opik_context_read_only_mode=True,
+        project_name=project_name,
+    )
+    return opik_tracer

opik/evaluation/models/__init__.py

@@ -1,7 +1,17 @@
 from .base_model import OpikBaseModel
 from .litellm.litellm_chat_model import LiteLLMChatModel
+from .langchain.langchain_chat_model import LangchainChatModel
+from .model_capabilities import (
+    MODEL_CAPABILITIES_REGISTRY,
+    ModelCapabilities,
+    ModelCapabilitiesRegistry,
+)
 
 __all__ = [
     "OpikBaseModel",
     "LiteLLMChatModel",
+    "LangchainChatModel",
+    "ModelCapabilities",
+    "ModelCapabilitiesRegistry",
+    "MODEL_CAPABILITIES_REGISTRY",
 ]