uipath 2.1.108__py3-none-any.whl → 2.1.109__py3-none-any.whl

uipath/eval/evaluators/contains_evaluator.py

@@ -0,0 +1,80 @@
+"""Contains evaluator for agent outputs."""
+
+from ..models import (
+    AgentExecution,
+    EvaluationResult,
+    EvaluatorType,
+    NumericEvaluationResult,
+)
+from .base_evaluator import BaseEvaluationCriteria
+from .output_evaluator import (
+    OutputEvaluator,
+    OutputEvaluatorConfig,
+)
+
+
+class ContainsEvaluationCriteria(BaseEvaluationCriteria):
+    """Evaluation criteria for the contains evaluator."""
+
+    search_text: str
+
+
+class ContainsEvaluatorConfig(OutputEvaluatorConfig[ContainsEvaluationCriteria]):
+    """Configuration for the contains evaluator."""
+
+    name: str = "ContainsEvaluator"
+    case_sensitive: bool = False
+    negated: bool = False
+
+
+class ContainsEvaluator(
+    OutputEvaluator[ContainsEvaluationCriteria, ContainsEvaluatorConfig, type(None)]  # type: ignore
+):
+    """Evaluator that checks if the actual output contains the expected output.
+
+    This evaluator returns True if the actual output contains the expected output,
+    and False otherwise. It supports case sensitivity and negation options.
+    """
+
+    @classmethod
+    def get_evaluator_id(cls) -> str:
+        """Get the evaluator id."""
+        return EvaluatorType.CONTAINS.value
+
+    async def evaluate(
+        self,
+        agent_execution: AgentExecution,
+        evaluation_criteria: ContainsEvaluationCriteria,
+    ) -> EvaluationResult:
+        """Evaluate whether actual output contains the expected output.
+
+        Args:
+            agent_execution: The execution details containing:
+                - agent_input: The input received by the agent
+                - agent_output: The actual output from the agent
+                - agent_trace: The execution spans to use for the evaluation
+            evaluation_criteria: The criteria to evaluate
+
+        Returns:
+            EvaluationResult: Boolean result indicating if output contains expected value (True/False)
+        """
+        actual_output = str(self._get_actual_output(agent_execution))
+        expected_output = str(self._get_expected_output(evaluation_criteria))
+
+        if not self.evaluator_config.case_sensitive:
+            actual_output = actual_output.lower()
+            expected_output = expected_output.lower()
+
+        is_contains = expected_output in actual_output
+
+        if self.evaluator_config.negated:
+            is_contains = not is_contains
+        return NumericEvaluationResult(
+            score=float(is_contains),
+        )
+
+    def _get_expected_output(
+        self, evaluation_criteria: ContainsEvaluationCriteria
+    ) -> str:
+        """Get the expected output from the evaluation criteria."""
+        return evaluation_criteria.search_text

uipath/eval/evaluators/exact_match_evaluator.py

@@ -1,14 +1,29 @@
-"""Exact match evaluator for binary pass/fail evaluation of agent outputs."""
+"""Exact match evaluator for agent outputs."""
 
-from typing import Any
+from ..models import (
+    AgentExecution,
+    EvaluationResult,
+    EvaluatorType,
+    NumericEvaluationResult,
+)
+from .output_evaluator import (
+    OutputEvaluationCriteria,
+    OutputEvaluator,
+    OutputEvaluatorConfig,
+)
 
-from uipath.eval.models import BooleanEvaluationResult, EvaluationResult
 
-from ..models.models import AgentExecution
-from .deterministic_evaluator_base import DeterministicEvaluatorBase
+class ExactMatchEvaluatorConfig(OutputEvaluatorConfig[OutputEvaluationCriteria]):
+    """Configuration for the exact match evaluator."""
 
+    name: str = "ExactMatchEvaluator"
+    case_sensitive: bool = False
+    negated: bool = False
 
-class ExactMatchEvaluator(DeterministicEvaluatorBase[dict[str, Any]]):
+
+class ExactMatchEvaluator(
+    OutputEvaluator[OutputEvaluationCriteria, ExactMatchEvaluatorConfig, type(None)]  # type: ignore
+):
     """Evaluator that performs exact structural matching between expected and actual outputs.
 
     This evaluator returns True if the actual output exactly matches the expected output
@@ -16,22 +31,38 @@ class ExactMatchEvaluator(DeterministicEvaluatorBase[dict[str, Any]]):
     to floats for consistent comparison.
     """
 
+    @classmethod
+    def get_evaluator_id(cls) -> str:
+        """Get the evaluator id."""
+        return EvaluatorType.EXACT_MATCH.value
+
     async def evaluate(
-        self, agent_execution: AgentExecution, evaluation_criteria: dict[str, Any]
+        self,
+        agent_execution: AgentExecution,
+        evaluation_criteria: OutputEvaluationCriteria,
     ) -> EvaluationResult:
         """Evaluate whether actual output exactly matches expected output.
 
         Args:
             agent_execution: The execution details containing:
                 - agent_input: The input received by the agent
-                - actual_output: The actual output from the agent
-                - spans: The execution spans to use for the evaluation
+                - agent_output: The actual output from the agent
+                - agent_trace: The execution spans to use for the evaluation
             evaluation_criteria: The criteria to evaluate
 
         Returns:
             EvaluationResult: Boolean result indicating exact match (True/False)
         """
-        return BooleanEvaluationResult(
-            score=self._canonical_json(agent_execution.agent_output)
-            == self._canonical_json(evaluation_criteria)
+        actual_output = str(self._get_actual_output(agent_execution))
+        expected_output = str(self._get_expected_output(evaluation_criteria))
+        if not self.evaluator_config.case_sensitive:
+            actual_output = actual_output.lower()
+            expected_output = expected_output.lower()
+
+        is_exact_match = actual_output == expected_output
+        if self.evaluator_config.negated:
+            is_exact_match = not is_exact_match
+
+        return NumericEvaluationResult(
+            score=float(is_exact_match),
         )

uipath/eval/evaluators/json_similarity_evaluator.py

@@ -1,17 +1,30 @@
 """JSON similarity evaluator for flexible structural comparison of outputs."""
 
 import math
-from typing import Any, Tuple, TypeVar
+from typing import Any, Tuple
 
-from uipath.eval.models import EvaluationResult, NumericEvaluationResult
+from ..models import (
+    AgentExecution,
+    EvaluationResult,
+    EvaluatorType,
+    NumericEvaluationResult,
+)
+from .output_evaluator import (
+    OutputEvaluationCriteria,
+    OutputEvaluator,
+    OutputEvaluatorConfig,
+)
 
-from ..models.models import AgentExecution
-from .deterministic_evaluator_base import DeterministicEvaluatorBase
 
-T = TypeVar("T")
+class JsonSimilarityEvaluatorConfig(OutputEvaluatorConfig[OutputEvaluationCriteria]):
+    """Configuration for the json similarity evaluator."""
 
+    name: str = "JsonSimilarityEvaluator"
 
-class JsonSimilarityEvaluator(DeterministicEvaluatorBase[dict[str, Any]]):
+
+class JsonSimilarityEvaluator(
+    OutputEvaluator[OutputEvaluationCriteria, JsonSimilarityEvaluatorConfig, str]
+):
     """Deterministic evaluator that scores structural JSON similarity between expected and actual output.
 
     Compares expected versus actual JSON-like structures and returns a
@@ -19,8 +32,15 @@ class JsonSimilarityEvaluator(DeterministicEvaluatorBase[dict[str, Any]]):
     and tolerant for numbers and strings (via Levenshtein distance).
     """
 
+    @classmethod
+    def get_evaluator_id(cls) -> str:
+        """Get the evaluator id."""
+        return EvaluatorType.JSON_SIMILARITY.value
+
     async def evaluate(
-        self, agent_execution: AgentExecution, evaluation_criteria: dict[str, Any]
+        self,
+        agent_execution: AgentExecution,
+        evaluation_criteria: OutputEvaluationCriteria,
     ) -> EvaluationResult:
         """Evaluate similarity between expected and actual JSON outputs.
 
@@ -36,16 +56,25 @@ class JsonSimilarityEvaluator(DeterministicEvaluatorBase[dict[str, Any]]):
         Returns:
             EvaluationResult: Numerical score between 0-100 indicating similarity
         """
+        score, justification = self._compare_json(
+            self._get_expected_output(evaluation_criteria),
+            self._get_actual_output(agent_execution),
+        )
+        validated_justification = self.validate_justification(justification)
         return NumericEvaluationResult(
-            score=self._compare_json(evaluation_criteria, agent_execution.agent_output)
+            score=score,
+            details=validated_justification,
         )
 
-    def _compare_json(self, expected: Any, actual: Any) -> float:
+    def _compare_json(self, expected: Any, actual: Any) -> tuple[float, str]:
         matched_leaves, total_leaves = self._compare_tokens(expected, actual)
         if total_leaves == 0:
-            return 100.0
-        sim = (matched_leaves / total_leaves) * 100.0
-        return max(0.0, min(100.0, sim))
+            return 1.0, "Total leaves are 0"
+        sim = matched_leaves / total_leaves
+        return (
+            max(0.0, min(1.0, sim)),
+            f"Matched leaves: {matched_leaves}, Total leaves: {total_leaves}",
+        )
 
     def _compare_tokens(
         self, expected_token: Any, actual_token: Any

uipath/eval/evaluators/legacy_base_evaluator.py

@@ -0,0 +1,89 @@
+"""Base evaluator abstract class for agent evaluation."""
+
+import functools
+import time
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from typing import Any, Generic, TypeVar
+
+from pydantic import BaseModel, ConfigDict
+
+from uipath.eval.models import EvaluationResult
+from uipath.eval.models.models import (
+    AgentExecution,
+    ErrorEvaluationResult,
+    LegacyEvaluatorCategory,
+    LegacyEvaluatorType,
+)
+
+
+def track_evaluation_metrics(func: Callable[..., Any]) -> Callable[..., Any]:
+    """Decorator to track evaluation metrics and handle errors gracefully."""
+
+    @functools.wraps(func)
+    async def wrapper(*args: Any, **kwargs: Any) -> EvaluationResult:
+        start_time = time.time()
+        try:
+            result = await func(*args, **kwargs)
+        except Exception as e:
+            result = ErrorEvaluationResult(
+                details="Exception thrown by evaluator: {}".format(e),
+                evaluation_time=time.time() - start_time,
+            )
+        end_time = time.time()
+        execution_time = end_time - start_time
+
+        result.evaluation_time = execution_time
+        return result
+
+    return wrapper
+
+
+T = TypeVar("T")
+
+
+class LegacyBaseEvaluator(BaseModel, Generic[T], ABC):
+    """Abstract base class for all evaluators."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    id: str
+    name: str
+    description: str
+    target_output_key: str = "*"
+    created_at: str
+    updated_at: str
+    category: LegacyEvaluatorCategory
+    evaluator_type: LegacyEvaluatorType
+
+    def __init_subclass__(cls, **kwargs: Any):
+        """Hook for subclass creation - automatically applies evaluation metrics tracking."""
+        super().__init_subclass__(**kwargs)
+
+        if hasattr(cls, "evaluate") and not getattr(
+            cls.evaluate, "_has_metrics_decorator", False
+        ):
+            cls.evaluate = track_evaluation_metrics(cls.evaluate)  # type: ignore[method-assign]
+            cls.evaluate._has_metrics_decorator = True  # type: ignore[attr-defined]
+
+    def model_post_init(self, __context: Any):
+        """Post-initialization hook for Pydantic models."""
+        pass
+
+    @abstractmethod
+    async def evaluate(
+        self, agent_execution: AgentExecution, evaluation_criteria: T
+    ) -> EvaluationResult:
+        """Evaluate the given data and return a result.
+
+        Args:
+            agent_execution: The execution details containing:
+                - agent_input: The input received by the agent
+                - actual_output: The actual output from the agent
+                - spans: The execution spans to use for the evaluation
+            evaluation_criteria: The criteria to evaluate
+
+        Returns:
+            EvaluationResult containing the score and details
+        """
+        pass

uipath/eval/evaluators/{deterministic_evaluator_base.py → legacy_deterministic_evaluator_base.py}

@@ -4,12 +4,12 @@ import json
 from abc import ABC
 from typing import Any, TypeVar
 
-from .base_evaluator import BaseEvaluator
+from .legacy_base_evaluator import LegacyBaseEvaluator
 
 T = TypeVar("T")
 
 
-class DeterministicEvaluatorBase(BaseEvaluator[T], ABC):
+class DeterministicEvaluatorBase(LegacyBaseEvaluator[T], ABC):
     """Base class for evaluators that produce deterministic, reproducible results.
 
     This class provides utility methods for canonical JSON comparison and number normalization

uipath/eval/evaluators/legacy_exact_match_evaluator.py

@@ -0,0 +1,37 @@
+"""Exact match evaluator for binary pass/fail evaluation of agent outputs."""
+
+from typing import Any
+
+from uipath.eval.models import BooleanEvaluationResult, EvaluationResult
+
+from ..models.models import AgentExecution
+from .legacy_deterministic_evaluator_base import DeterministicEvaluatorBase
+
+
+class LegacyExactMatchEvaluator(DeterministicEvaluatorBase[dict[str, Any]]):
+    """Evaluator that performs exact structural matching between expected and actual outputs.
+
+    This evaluator returns True if the actual output exactly matches the expected output
+    after canonical JSON normalization, and False otherwise. Numbers are normalized
+    to floats for consistent comparison.
+    """
+
+    async def evaluate(
+        self, agent_execution: AgentExecution, evaluation_criteria: dict[str, Any]
+    ) -> EvaluationResult:
+        """Evaluate whether actual output exactly matches expected output.
+
+        Args:
+            agent_execution: The execution details containing:
+                - agent_input: The input received by the agent
+                - actual_output: The actual output from the agent
+                - spans: The execution spans to use for the evaluation
+            evaluation_criteria: The criteria to evaluate
+
+        Returns:
+            EvaluationResult: Boolean result indicating exact match (True/False)
+        """
+        return BooleanEvaluationResult(
+            score=self._canonical_json(agent_execution.agent_output)
+            == self._canonical_json(evaluation_criteria)
+        )

uipath/eval/evaluators/legacy_json_similarity_evaluator.py

@@ -0,0 +1,151 @@
+"""JSON similarity evaluator for flexible structural comparison of outputs."""
+
+import math
+from typing import Any, Tuple, TypeVar
+
+from uipath.eval.models import EvaluationResult, NumericEvaluationResult
+
+from ..models.models import AgentExecution
+from .legacy_deterministic_evaluator_base import DeterministicEvaluatorBase
+
+T = TypeVar("T")
+
+
+class LegacyJsonSimilarityEvaluator(DeterministicEvaluatorBase[dict[str, Any]]):
+    """Legacy deterministic evaluator that scores structural JSON similarity between expected and actual output.
+
+    Compares expected versus actual JSON-like structures and returns a
+    numerical score in the range [0, 100]. The comparison is token-based
+    and tolerant for numbers and strings (via Levenshtein distance).
+    """
+
+    async def evaluate(
+        self, agent_execution: AgentExecution, evaluation_criteria: dict[str, Any]
+    ) -> EvaluationResult:
+        """Evaluate similarity between expected and actual JSON outputs.
+
+        Uses token-based comparison with tolerance for numeric differences
+        and Levenshtein distance for string similarity.
+
+            agent_execution: The execution details containing:
+                - agent_input: The input received by the agent
+                - actual_output: The actual output from the agent
+                - spans: The execution spans to use for the evaluation
+            evaluation_criteria: The criteria to evaluate
+
+        Returns:
+            EvaluationResult: Numerical score between 0-100 indicating similarity
+        """
+        return NumericEvaluationResult(
+            score=self._compare_json(evaluation_criteria, agent_execution.agent_output)
+        )
+
+    def _compare_json(self, expected: Any, actual: Any) -> float:
+        matched_leaves, total_leaves = self._compare_tokens(expected, actual)
+        if total_leaves == 0:
+            return 100.0
+        sim = (matched_leaves / total_leaves) * 100.0
+        return max(0.0, min(100.0, sim))
+
+    def _compare_tokens(
+        self, expected_token: Any, actual_token: Any
+    ) -> Tuple[float, float]:
+        if self._is_number(expected_token) and self._is_number(actual_token):
+            return self._compare_numbers(float(expected_token), float(actual_token))
+
+        if type(expected_token) is not type(actual_token):
+            return 0.0, self._count_leaves(expected_token)
+
+        if isinstance(expected_token, dict):
+            matched_leaves = total_leaves = 0.0
+            # Only expected keys count
+            for expected_key, expected_value in expected_token.items():
+                if isinstance(actual_token, dict) and expected_key in actual_token:
+                    matched, total = self._compare_tokens(
+                        expected_value, actual_token[expected_key]
+                    )
+                else:
+                    matched, total = (0.0, self._count_leaves(expected_value))
+                matched_leaves += matched
+                total_leaves += total
+            return matched_leaves, total_leaves
+
+        if isinstance(expected_token, list):
+            matched_leaves = total_leaves = 0.0
+            common_length = min(len(expected_token), len(actual_token))
+            for index in range(common_length):
+                matched, total = self._compare_tokens(
+                    expected_token[index], actual_token[index]
+                )
+                matched_leaves += matched
+                total_leaves += total
+            for index in range(common_length, len(expected_token)):
+                total_leaves += self._count_leaves(expected_token[index])
+            return (matched_leaves, total_leaves)
+
+        if isinstance(expected_token, bool):
+            return (1.0, 1.0) if expected_token == actual_token else (0.0, 1.0)
+
+        if isinstance(expected_token, str):
+            return self._compare_strings(expected_token, actual_token)
+
+        return (1.0, 1.0) if str(expected_token) == str(actual_token) else (0.0, 1.0)
+
+    def _compare_numbers(
+        self, expected_number: float, actual_number: float
+    ) -> Tuple[float, float]:
+        total = 1.0
+        if math.isclose(expected_number, 0.0, abs_tol=1e-12):
+            matched = 1.0 if math.isclose(actual_number, 0.0, abs_tol=1e-12) else 0.0
+        else:
+            ratio = abs(expected_number - actual_number) / abs(expected_number)
+            matched = max(0.0, min(1.0, 1.0 - ratio))
+        return matched, total
+
+    def _compare_strings(
+        self, expected_string: str, actual_string: str
+    ) -> Tuple[float, float]:
+        total = 1.0
+        if not expected_string and not actual_string:
+            return 1.0, total
+        distance = self._levenshtein(expected_string, actual_string)
+        max_length = max(len(expected_string), len(actual_string))
+        similarity = 1.0 - (distance / max_length) if max_length else 1.0
+        similarity = max(0.0, min(1.0, similarity))
+        return similarity, total
+
+    def _count_leaves(self, token_node: Any) -> float:
+        if isinstance(token_node, dict):
+            return sum(
+                self._count_leaves(child_value) for child_value in token_node.values()
+            )
+        if isinstance(token_node, list):
+            return sum(self._count_leaves(child_value) for child_value in token_node)
+        return 1.0
+
+    def _levenshtein(self, source_text: str, target_text: str) -> int:
+        if not source_text:
+            return len(target_text)
+        if not target_text:
+            return len(source_text)
+        source_len, target_len = len(source_text), len(target_text)
+        distance_matrix = [[0] * (target_len + 1) for _ in range(source_len + 1)]
+        for row_idx in range(source_len + 1):
+            distance_matrix[row_idx][0] = row_idx
+        for col_idx in range(target_len + 1):
+            distance_matrix[0][col_idx] = col_idx
+        for row_idx in range(1, source_len + 1):
+            for col_idx in range(1, target_len + 1):
+                substitution_cost = (
+                    0 if source_text[row_idx - 1] == target_text[col_idx - 1] else 1
+                )
+                distance_matrix[row_idx][col_idx] = min(
+                    distance_matrix[row_idx - 1][col_idx] + 1,  # deletion
+                    distance_matrix[row_idx][col_idx - 1] + 1,  # insertion
+                    distance_matrix[row_idx - 1][col_idx - 1]
+                    + substitution_cost,  # substitution
+                )
+        return distance_matrix[source_len][target_len]
+
+    def _is_number(self, value: Any) -> bool:
+        return isinstance(value, (int, float)) and not isinstance(value, bool)

uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py

@@ -0,0 +1,137 @@
+"""LLM-as-a-judge evaluator for subjective quality assessment of agent outputs."""
+
+import json
+from typing import Any, Optional
+
+from pydantic import field_validator
+
+from uipath.eval.models import NumericEvaluationResult
+
+from ..._services import UiPathLlmChatService
+from ..._utils.constants import COMMUNITY_agents_SUFFIX
+from ..models.models import AgentExecution, EvaluationResult, LLMResponse
+from .legacy_base_evaluator import LegacyBaseEvaluator
+
+
+class LegacyLlmAsAJudgeEvaluator(LegacyBaseEvaluator[dict[str, Any]]):
+    """Legacy evaluator that uses an LLM to judge the quality of agent output."""
+
+    prompt: str
+    model: str
+    actual_output_placeholder: str = "{{ActualOutput}}"
+    expected_output_placeholder: str = "{{ExpectedOutput}}"
+    llm: Optional[UiPathLlmChatService] = None
+
+    @field_validator("prompt")
+    @classmethod
+    def validate_prompt_placeholders(cls, v: str) -> str:
+        """Validate that prompt contains required placeholders."""
+        if "{{ActualOutput}}" not in v or "{{ExpectedOutput}}" not in v:
+            raise ValueError(
+                "Prompt must contain both {ActualOutput} and {ExpectedOutput} placeholders"
+            )
+        return v
+
+    def model_post_init(self, __context: Any):
+        """Initialize the LLM service after model creation."""
+        super().model_post_init(__context)
+        self._initialize_llm()
+
+    def _initialize_llm(self):
+        """Initialize the LLM used for evaluation."""
+        from uipath import UiPath
+
+        uipath = UiPath()
+        self.llm = uipath.llm
+
+    async def evaluate(
+        self,
+        agent_execution: AgentExecution,
+        evaluation_criteria: dict[str, Any],
+    ) -> EvaluationResult:
+        """Evaluate using an LLM as a judge.
+
+        Sends the formatted prompt to the configured LLM and expects a JSON response
+        with a numerical score (0-100) and justification.
+
+            agent_execution: The execution details containing:
+                - agent_input: The input received by the agent
+                - actual_output: The actual output from the agent
+                - spans: The execution spans to use for the evaluation
+            evaluation_criteria: The criteria to evaluate
+
+        Returns:
+            EvaluationResult: Numerical score with LLM justification as details
+        """
+        # Create the evaluation prompt
+        evaluation_prompt = self._create_evaluation_prompt(
+            expected_output=evaluation_criteria,
+            actual_output=agent_execution.agent_output,
+        )
+
+        llm_response = await self._get_llm_response(evaluation_prompt)
+
+        return NumericEvaluationResult(
+            score=llm_response.score,
+            details=llm_response.justification,
+        )
+
+    def _create_evaluation_prompt(
+        self, expected_output: Any, actual_output: Any
+    ) -> str:
+        """Create the evaluation prompt for the LLM."""
+        formatted_prompt = self.prompt.replace(
+            self.actual_output_placeholder,
+            str(actual_output),
+        )
+        formatted_prompt = formatted_prompt.replace(
+            self.expected_output_placeholder,
+            str(expected_output),
+        )
+
+        return formatted_prompt
+
+    async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse:
+        """Get response from the LLM.
+
+        Args:
+            evaluation_prompt: The formatted prompt to send to the LLM
+
+        Returns:
+            LLMResponse with score and justification
+        """
+        # remove community-agents suffix from llm model name
+        model = self.model
+        if model.endswith(COMMUNITY_agents_SUFFIX):
+            model = model.replace(COMMUNITY_agents_SUFFIX, "")
+
+        # Prepare the request
+        request_data = {
+            "model": model,
+            "messages": [{"role": "user", "content": evaluation_prompt}],
+            "response_format": {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": "evaluation_response",
+                    "schema": {
+                        "type": "object",
+                        "properties": {
+                            "score": {
+                                "type": "number",
+                                "minimum": 0,
+                                "maximum": 100,
+                                "description": "Score between 0 and 100",
+                            },
+                            "justification": {
+                                "type": "string",
+                                "description": "Explanation for the score",
+                            },
+                        },
+                        "required": ["score", "justification"],
+                    },
+                },
+            },
+        }
+
+        response = await self.llm.chat_completions(**request_data)  # type: ignore
+        return LLMResponse(**json.loads(response.choices[-1].message.content or "{}"))