uipath 2.1.108__py3-none-any.whl → 2.1.110__py3-none-any.whl

uipath/eval/evaluators/output_evaluator.py

@@ -0,0 +1,117 @@
+"""Base class for all output evaluator configurations."""
+
+import json
+from typing import Any, TypeVar, Union
+
+from pydantic import Field
+
+from ..models import AgentExecution
+from ..models.models import UiPathEvaluationError, UiPathEvaluationErrorCategory
+from .base_evaluator import (
+    BaseEvaluationCriteria,
+    BaseEvaluator,
+    BaseEvaluatorConfig,
+    BaseEvaluatorJustification,
+)
+
+
+class OutputEvaluationCriteria(BaseEvaluationCriteria):
+    """Base class for all output evaluation criteria."""
+
+    expected_output: dict[str, Any] | str
+
+
+T = TypeVar("T", bound=BaseEvaluationCriteria)
+T_OutputCriteria = TypeVar("T_OutputCriteria", bound=OutputEvaluationCriteria)
+
+
+class OutputEvaluatorConfig(BaseEvaluatorConfig[T]):
+    """Base class for all output evaluator configurations.
+
+    Generic over T to allow subclasses to define their own
+    specific output evaluation criteria types while maintaining type safety.
+    """
+
+    target_output_key: str = Field(
+        default="*", description="Key to extract output from agent execution"
+    )
+
+
+C = TypeVar("C", bound=OutputEvaluatorConfig[Any])
+J = TypeVar("J", bound=Union[str, None, BaseEvaluatorJustification])
+
+
+class BaseOutputEvaluator(BaseEvaluator[T, C, J]):
+    """Abstract base class for all output evaluators.
+
+    Generic Parameters:
+        T_OutputCriteria: The output evaluation criteria type
+        C: The output evaluator config type (bound to OutputEvaluatorConfig[T_OutputCriteria])
+        J: The justification type
+    """
+
+    def _get_actual_output(self, agent_execution: AgentExecution) -> Any:
+        """Get the actual output from the agent execution."""
+        if self.evaluator_config.target_output_key != "*":
+            try:
+                return agent_execution.agent_output[
+                    self.evaluator_config.target_output_key
+                ]
+            except KeyError as e:
+                raise UiPathEvaluationError(
+                    code="TARGET_OUTPUT_KEY_NOT_FOUND",
+                    title="Target output key not found in actual output",
+                    detail=f"Error: {e}",
+                    category=UiPathEvaluationErrorCategory.USER,
+                ) from e
+        return agent_execution.agent_output
+
+    def _get_full_expected_output(self, evaluation_criteria: T) -> Any:
+        """Get the full expected output from the evaluation criteria."""
+        raise UiPathEvaluationError(
+            code="NOT_IMPLEMENTED",
+            title="This method was not implemented by the subclass.",
+            detail="This method was not implemented by the subclass.",
+            category=UiPathEvaluationErrorCategory.SYSTEM,
+        )
+
+    def _get_expected_output(self, evaluation_criteria: T) -> Any:
+        """Load the expected output from the evaluation criteria."""
+        expected_output = self._get_full_expected_output(evaluation_criteria)
+        if self.evaluator_config.target_output_key != "*":
+            if isinstance(expected_output, str):
+                try:
+                    expected_output = json.loads(expected_output)
+                except json.JSONDecodeError as e:
+                    raise UiPathEvaluationError(
+                        code="INVALID_EXPECTED_OUTPUT",
+                        title="When target output key is not '*', expected output must be a dictionary or a valid JSON string",
+                        detail=f"Error: {e}",
+                        category=UiPathEvaluationErrorCategory.USER,
+                    ) from e
+            try:
+                expected_output = expected_output[
+                    self.evaluator_config.target_output_key
+                ]
+            except KeyError as e:
+                raise UiPathEvaluationError(
+                    code="TARGET_OUTPUT_KEY_NOT_FOUND",
+                    title="Target output key not found in expected output",
+                    detail=f"Error: {e}",
+                    category=UiPathEvaluationErrorCategory.USER,
+                ) from e
+        return expected_output
+
+
+class OutputEvaluator(BaseOutputEvaluator[T_OutputCriteria, C, J]):
+    """Abstract base class for all output evaluators.
+
+    Generic Parameters:
+        T_OutputCriteria: The output evaluation criteria type
+        C: The output evaluator config type (bound to OutputEvaluatorConfig[T_OutputCriteria])
+        J: The justification type
+    """
+
+    def _get_full_expected_output(self, evaluation_criteria: T_OutputCriteria) -> Any:
+        """Get the full expected output from the evaluation criteria."""
+        return evaluation_criteria.expected_output

uipath/eval/evaluators/tool_call_args_evaluator.py

@@ -0,0 +1,82 @@
+"""Tool call order evaluator for validating correct sequence of tool calls."""
+
+from .._helpers.evaluators_helpers import (
+    extract_tool_calls,
+    tool_calls_args_score,
+)
+from ..models import AgentExecution, EvaluationResult, NumericEvaluationResult, ToolCall
+from ..models.models import EvaluatorType
+from .base_evaluator import (
+    BaseEvaluationCriteria,
+    BaseEvaluator,
+    BaseEvaluatorConfig,
+    BaseEvaluatorJustification,
+)
+
+
+class ToolCallArgsEvaluationCriteria(BaseEvaluationCriteria):
+    """Evaluation criteria for the tool call order evaluator."""
+
+    # TODO: name field of ToolCall needs to be validated such that it contains only the tools available
+    tool_calls: list[ToolCall]
+
+
+class ToolCallArgsEvaluatorConfig(BaseEvaluatorConfig[ToolCallArgsEvaluationCriteria]):
+    """Configuration for the tool call count evaluator."""
+
+    name: str = "ToolCallArgsEvaluator"
+    strict: bool = False
+    subset: bool = False
+
+
+class ToolCallArgsEvaluatorJustification(BaseEvaluatorJustification):
+    """Justification for the tool call args evaluator."""
+
+    explained_tool_calls_args: dict[str, str]
+
+
+class ToolCallArgsEvaluator(
+    BaseEvaluator[
+        ToolCallArgsEvaluationCriteria,
+        ToolCallArgsEvaluatorConfig,
+        ToolCallArgsEvaluatorJustification,
+    ]
+):
+    """Evaluator that checks if the tool calls are in the correct order.
+
+    This evaluator returns True if the tool calls are in the correct order, and False otherwise.
+    """
+
+    @classmethod
+    def get_evaluator_id(cls) -> str:
+        """Get the evaluator id."""
+        return EvaluatorType.TOOL_CALL_ARGS.value
+
+    async def evaluate(
+        self,
+        agent_execution: AgentExecution,
+        evaluation_criteria: ToolCallArgsEvaluationCriteria,
+    ) -> EvaluationResult:
+        """Evaluate if the tool calls are in the correct order.
+
+        Args:
+            agent_execution: The execution details containing:
+                - agent_input: The input received by the agent
+                - agent_output: The final output of the agent
+                - agent_trace: The execution spans to use for the evaluation
+            evaluation_criteria: The criteria to evaluate
+        Returns:
+            EvaluationResult: Boolean result indicating correct tool call order (True/False)
+        """
+        tool_calls_order = extract_tool_calls(agent_execution.agent_trace)
+        score, justification = tool_calls_args_score(
+            tool_calls_order,
+            evaluation_criteria.tool_calls,
+            self.evaluator_config.strict,
+            self.evaluator_config.subset,
+        )
+        validated_justification = self.validate_justification(justification)
+        return NumericEvaluationResult(
+            score=score,
+            details=validated_justification,
+        )

uipath/eval/evaluators/tool_call_count_evaluator.py

@@ -0,0 +1,87 @@
+"""Tool call count evaluator for validating expected tool usage patterns."""
+
+from collections import Counter
+
+from .._helpers.evaluators_helpers import (
+    extract_tool_calls_names,
+    tool_calls_count_score,
+)
+from ..models import AgentExecution, EvaluationResult, NumericEvaluationResult
+from ..models.models import EvaluatorType
+from .base_evaluator import (
+    BaseEvaluationCriteria,
+    BaseEvaluator,
+    BaseEvaluatorConfig,
+    BaseEvaluatorJustification,
+)
+
+
+class ToolCallCountEvaluationCriteria(BaseEvaluationCriteria):
+    """Evaluation criteria for the tool call count evaluator."""
+
+    # TODO: str field needs to be validated against some criteria that allows ">x", "<x", ">=x", "<=x", "x"
+    tool_calls_count: dict[str, tuple[str, int]]
+
+
+class ToolCallCountEvaluatorConfig(
+    BaseEvaluatorConfig[ToolCallCountEvaluationCriteria]
+):
+    """Configuration for the tool call count evaluator."""
+
+    name: str = "ToolCallCountEvaluator"
+    strict: bool = False
+
+
+class ToolCallCountEvaluatorJustification(BaseEvaluatorJustification):
+    """Justification for the tool call count evaluator."""
+
+    explained_tool_calls_count: dict[str, str]
+
+
+class ToolCallCountEvaluator(
+    BaseEvaluator[
+        ToolCallCountEvaluationCriteria,
+        ToolCallCountEvaluatorConfig,
+        ToolCallCountEvaluatorJustification,
+    ]
+):
+    """Evaluator that checks if the tool calls match the expected count.
+
+    This evaluator returns a score based on how well the actual tool call counts
+    match the expected counts specified in the criteria.
+    """
+
+    @classmethod
+    def get_evaluator_id(cls) -> str:
+        """Get the evaluator id."""
+        return EvaluatorType.TOOL_CALL_COUNT.value
+
+    async def evaluate(
+        self,
+        agent_execution: AgentExecution,
+        evaluation_criteria: ToolCallCountEvaluationCriteria,
+    ) -> EvaluationResult:
+        """Evaluate if the tool calls are in the correct order.
+
+        Args:
+            agent_execution: The execution details containing:
+                - agent_input: The input received by the agent
+                - agent_output: The final output of the agent
+                - agent_trace: The execution spans to use for the evaluation
+            evaluation_criteria: The criteria to evaluate
+        Returns:
+            EvaluationResult: Boolean result indicating correct tool call order (True/False)
+        """
+        tool_calls_count = Counter(
+            extract_tool_calls_names(agent_execution.agent_trace)
+        )
+        score, justification = tool_calls_count_score(
+            tool_calls_count,
+            evaluation_criteria.tool_calls_count,
+            self.evaluator_config.strict,
+        )
+        validated_justification = self.validate_justification(justification)
+        return NumericEvaluationResult(
+            score=score,
+            details=validated_justification,
+        )

uipath/eval/evaluators/tool_call_order_evaluator.py

@@ -0,0 +1,84 @@
+"""Tool call order evaluator for validating correct sequence of tool calls."""
+
+from .._helpers.evaluators_helpers import (
+    extract_tool_calls_names,
+    tool_calls_order_score,
+)
+from ..models import AgentExecution, EvaluationResult, NumericEvaluationResult
+from ..models.models import EvaluatorType
+from .base_evaluator import (
+    BaseEvaluationCriteria,
+    BaseEvaluator,
+    BaseEvaluatorConfig,
+    BaseEvaluatorJustification,
+)
+
+
+class ToolCallOrderEvaluationCriteria(BaseEvaluationCriteria):
+    """Evaluation criteria for the tool call order evaluator."""
+
+    # TODO: str field needs to be validated such that it contains only the tools available
+    tool_calls_order: list[str]
+
+
+class ToolCallOrderEvaluatorConfig(
+    BaseEvaluatorConfig[ToolCallOrderEvaluationCriteria]
+):
+    """Configuration for the tool call count evaluator."""
+
+    name: str = "ToolCallOrderEvaluator"
+    strict: bool = False
+
+
+class ToolCallOrderEvaluatorJustification(BaseEvaluatorJustification):
+    """Justification for the tool call order evaluator."""
+
+    actual_tool_calls_order: list[str]
+    expected_tool_calls_order: list[str]
+    lcs: list[str]
+
+
+class ToolCallOrderEvaluator(
+    BaseEvaluator[
+        ToolCallOrderEvaluationCriteria,
+        ToolCallOrderEvaluatorConfig,
+        ToolCallOrderEvaluatorJustification,
+    ]
+):
+    """Evaluator that checks if the tool calls are in the correct order.
+
+    This evaluator returns True if the tool calls are in the correct order, and False otherwise.
+    """
+
+    @classmethod
+    def get_evaluator_id(cls) -> str:
+        """Get the evaluator id."""
+        return EvaluatorType.TOOL_CALL_ORDER.value
+
+    async def evaluate(
+        self,
+        agent_execution: AgentExecution,
+        evaluation_criteria: ToolCallOrderEvaluationCriteria,
+    ) -> EvaluationResult:
+        """Evaluate if the tool calls are in the correct order.
+
+        Args:
+            agent_execution: The execution details containing:
+                - agent_input: The input received by the agent
+                - agent_output: The final output of the agent
+                - agent_trace: The execution spans to use for the evaluation
+            evaluation_criteria: The criteria to evaluate
+        Returns:
+            EvaluationResult: Boolean result indicating correct tool call order (True/False)
+        """
+        tool_calls_order = extract_tool_calls_names(agent_execution.agent_trace)
+        score, justification = tool_calls_order_score(
+            tool_calls_order,
+            evaluation_criteria.tool_calls_order,
+            self.evaluator_config.strict,
+        )
+        validated_justification = self.validate_justification(justification)
+        return NumericEvaluationResult(
+            score=score,
+            details=validated_justification,
+        )

uipath/eval/evaluators/tool_call_output_evaluator.py

@@ -0,0 +1,87 @@
+"""Tool call order evaluator for validating correct sequence of tool calls."""
+
+from .._helpers.evaluators_helpers import (
+    extract_tool_calls_outputs,
+    tool_calls_output_score,
+)
+from ..models import (
+    AgentExecution,
+    EvaluationResult,
+    NumericEvaluationResult,
+    ToolOutput,
+)
+from ..models.models import EvaluatorType
+from .base_evaluator import (
+    BaseEvaluationCriteria,
+    BaseEvaluator,
+    BaseEvaluatorConfig,
+    BaseEvaluatorJustification,
+)
+
+
+class ToolCallOutputEvaluationCriteria(BaseEvaluationCriteria):
+    """Evaluation criteria for the tool call order evaluator."""
+
+    # TODO: name field of ToolCall needs to be validated such that it contains only the tools available
+    tool_outputs: list[ToolOutput]
+
+
+class ToolCallOutputEvaluatorConfig(
+    BaseEvaluatorConfig[ToolCallOutputEvaluationCriteria]
+):
+    """Configuration for the tool call count evaluator."""
+
+    name: str = "ToolCallOutputEvaluator"
+    strict: bool = False
+
+
+class ToolCallOutputEvaluatorJustification(BaseEvaluatorJustification):
+    """Justification for the tool call output evaluator."""
+
+    explained_tool_calls_outputs: dict[str, str]
+
+
+class ToolCallOutputEvaluator(
+    BaseEvaluator[
+        ToolCallOutputEvaluationCriteria,
+        ToolCallOutputEvaluatorConfig,
+        ToolCallOutputEvaluatorJustification,
+    ]
+):
+    """Evaluator that checks if the tool calls are in the correct order.
+
+    This evaluator returns True if the tool calls are in the correct order, and False otherwise.
+    """
+
+    @classmethod
+    def get_evaluator_id(cls) -> str:
+        """Get the evaluator id."""
+        return EvaluatorType.TOOL_CALL_OUTPUT.value
+
+    async def evaluate(
+        self,
+        agent_execution: AgentExecution,
+        evaluation_criteria: ToolCallOutputEvaluationCriteria,
+    ) -> EvaluationResult:
+        """Evaluate if the tool calls are in the correct order.
+
+        Args:
+            agent_execution: The execution details containing:
+                - agent_input: The input received by the agent
+                - agent_output: The final output of the agent
+                - agent_trace: The execution spans to use for the evaluation
+            evaluation_criteria: The criteria to evaluate
+        Returns:
+            EvaluationResult: Boolean result indicating correct tool call order (True/False)
+        """
+        tool_calls_outputs = extract_tool_calls_outputs(agent_execution.agent_trace)
+        score, justification = tool_calls_output_score(
+            tool_calls_outputs,
+            evaluation_criteria.tool_outputs,
+            self.evaluator_config.strict,
+        )
+        validated_justification = self.validate_justification(justification)
+        return NumericEvaluationResult(
+            score=score,
+            details=validated_justification,
+        )

uipath/eval/evaluators_types/ContainsEvaluator.json

@@ -0,0 +1,73 @@
+{
+  "evaluatorTypeId": "uipath-contains",
+  "evaluatorConfigSchema": {
+    "$defs": {
+      "ContainsEvaluationCriteria": {
+        "description": "Evaluation criteria for the contains evaluator.",
+        "properties": {
+          "search_text": {
+            "title": "Search Text",
+            "type": "string"
+          }
+        },
+        "required": [
+          "search_text"
+        ],
+        "title": "ContainsEvaluationCriteria",
+        "type": "object"
+      }
+    },
+    "description": "Configuration for the contains evaluator.",
+    "properties": {
+      "name": {
+        "default": "ContainsEvaluator",
+        "title": "Name",
+        "type": "string"
+      },
+      "default_evaluation_criteria": {
+        "anyOf": [
+          {
+            "$ref": "#/$defs/ContainsEvaluationCriteria"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "default": null
+      },
+      "target_output_key": {
+        "default": "*",
+        "description": "Key to extract output from agent execution",
+        "title": "Target Output Key",
+        "type": "string"
+      },
+      "case_sensitive": {
+        "default": false,
+        "title": "Case Sensitive",
+        "type": "boolean"
+      },
+      "negated": {
+        "default": false,
+        "title": "Negated",
+        "type": "boolean"
+      }
+    },
+    "title": "ContainsEvaluatorConfig",
+    "type": "object"
+  },
+  "evaluationCriteriaSchema": {
+    "description": "Evaluation criteria for the contains evaluator.",
+    "properties": {
+      "search_text": {
+        "title": "Search Text",
+        "type": "string"
+      }
+    },
+    "required": [
+      "search_text"
+    ],
+    "title": "ContainsEvaluationCriteria",
+    "type": "object"
+  },
+  "justificationSchema": {}
+}

uipath/eval/evaluators_types/ExactMatchEvaluator.json

@@ -0,0 +1,89 @@
+{
+  "evaluatorTypeId": "uipath-exact-match",
+  "evaluatorConfigSchema": {
+    "$defs": {
+      "OutputEvaluationCriteria": {
+        "description": "Base class for all output evaluation criteria.",
+        "properties": {
+          "expected_output": {
+            "anyOf": [
+              {
+                "additionalProperties": true,
+                "type": "object"
+              },
+              {
+                "type": "string"
+              }
+            ],
+            "title": "Expected Output"
+          }
+        },
+        "required": [
+          "expected_output"
+        ],
+        "title": "OutputEvaluationCriteria",
+        "type": "object"
+      }
+    },
+    "description": "Configuration for the exact match evaluator.",
+    "properties": {
+      "name": {
+        "default": "ExactMatchEvaluator",
+        "title": "Name",
+        "type": "string"
+      },
+      "default_evaluation_criteria": {
+        "anyOf": [
+          {
+            "$ref": "#/$defs/OutputEvaluationCriteria"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "default": null
+      },
+      "target_output_key": {
+        "default": "*",
+        "description": "Key to extract output from agent execution",
+        "title": "Target Output Key",
+        "type": "string"
+      },
+      "case_sensitive": {
+        "default": false,
+        "title": "Case Sensitive",
+        "type": "boolean"
+      },
+      "negated": {
+        "default": false,
+        "title": "Negated",
+        "type": "boolean"
+      }
+    },
+    "title": "ExactMatchEvaluatorConfig",
+    "type": "object"
+  },
+  "evaluationCriteriaSchema": {
+    "description": "Base class for all output evaluation criteria.",
+    "properties": {
+      "expected_output": {
+        "anyOf": [
+          {
+            "additionalProperties": true,
+            "type": "object"
+          },
+          {
+            "type": "string"
+          }
+        ],
+        "title": "Expected Output"
+      }
+    },
+    "required": [
+      "expected_output"
+    ],
+    "title": "OutputEvaluationCriteria",
+    "type": "object"
+  },
+  "justificationSchema": {}
+}