uipath 2.1.52__py3-none-any.whl → 2.1.53__py3-none-any.whl

uipath/eval/evaluators/base_evaluator.py

@@ -0,0 +1,88 @@
+"""Base evaluator abstract class for agent evaluation."""
+
+import functools
+import time
+from abc import ABC, abstractmethod
+from typing import Generic, TypeVar
+
+from pydantic import BaseModel, ConfigDict
+
+from uipath.eval.models import EvaluationResult
+from uipath.eval.models.models import (
+    AgentExecution,
+    ErrorEvaluationResult,
+    EvaluatorCategory,
+    EvaluatorType,
+)
+
+
+def track_evaluation_metrics(func):
+    """Decorator to track evaluation metrics and handle errors gracefully."""
+
+    @functools.wraps(func)
+    async def wrapper(*args, **kwargs) -> EvaluationResult:
+        start_time = time.time()
+        try:
+            result = await func(*args, **kwargs)
+        except Exception as e:
+            result = ErrorEvaluationResult(
+                details="Exception thrown by evaluator: {}".format(e),
+                evaluation_time=time.time() - start_time,
+            )
+        end_time = time.time()
+        execution_time = end_time - start_time
+
+        result.evaluation_time = execution_time
+        return result
+
+    return wrapper
+
+
+T = TypeVar("T")
+
+
+class BaseEvaluator(BaseModel, Generic[T], ABC):
+    """Abstract base class for all evaluators."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    id: str
+    name: str
+    description: str
+    target_output_key: str = "*"
+    created_at: str
+    updated_at: str
+    category: EvaluatorCategory
+    evaluator_type: EvaluatorType
+
+    def __init_subclass__(cls, **kwargs):
+        """Hook for subclass creation - automatically applies evaluation metrics tracking."""
+        super().__init_subclass__(**kwargs)
+
+        if hasattr(cls, "evaluate") and not getattr(
+            cls.evaluate, "_has_metrics_decorator", False
+        ):
+            cls.evaluate = track_evaluation_metrics(cls.evaluate)  # type: ignore[method-assign]
+            cls.evaluate._has_metrics_decorator = True  # type: ignore[attr-defined]
+
+    def model_post_init(self, __context):
+        """Post-initialization hook for Pydantic models."""
+        pass
+
+    @abstractmethod
+    async def evaluate(
+        self, agent_execution: AgentExecution, evaluation_criteria: T
+    ) -> EvaluationResult:
+        """Evaluate the given data and return a result.
+
+        Args:
+            agent_execution: The execution details containing:
+                - agent_input: The input received by the agent
+                - actual_output: The actual output from the agent
+                - spans: The execution spans to use for the evaluation
+            evaluation_criteria: The criteria to evaluate
+
+        Returns:
+            EvaluationResult containing the score and details
+        """
+        pass

uipath/eval/evaluators/deterministic_evaluator_base.py

@@ -0,0 +1,53 @@
+"""Base class for deterministic evaluators that provide consistent outputs."""
+
+import json
+from abc import ABC
+from typing import Any, TypeVar
+
+from .base_evaluator import BaseEvaluator
+
+T = TypeVar("T")
+
+
+class DeterministicEvaluatorBase(BaseEvaluator[T], ABC):
+    """Base class for evaluators that produce deterministic, reproducible results.
+
+    This class provides utility methods for canonical JSON comparison and number normalization
+    to ensure consistent evaluation results across runs.
+    """
+
+    def _canonical_json(self, obj: Any) -> str:
+        """Convert an object to canonical JSON string for consistent comparison.
+
+        Args:
+            obj: The object to convert to canonical JSON
+
+        Returns:
+            str: Canonical JSON string with normalized numbers and sorted keys
+        """
+        return json.dumps(
+            self._normalize_numbers(obj),
+            sort_keys=True,
+            separators=(",", ":"),
+            ensure_ascii=False,
+        )
+
+    def _normalize_numbers(self, obj: Any) -> Any:
+        """Recursively normalize numbers in nested data structures.
+
+        Converts all numeric values (int, float) to float for consistent comparison,
+        while preserving booleans and other data types.
+
+        Args:
+            obj: The object to normalize
+
+        Returns:
+            Any: Object with normalized numbers
+        """
+        if isinstance(obj, dict):
+            return {k: self._normalize_numbers(v) for k, v in obj.items()}
+        if isinstance(obj, (list, tuple)):
+            return [self._normalize_numbers(v) for v in obj]
+        if isinstance(obj, (int, float)) and not isinstance(obj, bool):
+            return float(obj)
+        return obj

uipath/eval/evaluators/exact_match_evaluator.py

@@ -0,0 +1,37 @@
+"""Exact match evaluator for binary pass/fail evaluation of agent outputs."""
+
+from typing import Any
+
+from uipath.eval.models import BooleanEvaluationResult, EvaluationResult
+
+from ..models.models import AgentExecution
+from .deterministic_evaluator_base import DeterministicEvaluatorBase
+
+
+class ExactMatchEvaluator(DeterministicEvaluatorBase[dict[str, Any]]):
+    """Evaluator that performs exact structural matching between expected and actual outputs.
+
+    This evaluator returns True if the actual output exactly matches the expected output
+    after canonical JSON normalization, and False otherwise. Numbers are normalized
+    to floats for consistent comparison.
+    """
+
+    async def evaluate(
+        self, agent_execution: AgentExecution, evaluation_criteria: dict[str, Any]
+    ) -> EvaluationResult:
+        """Evaluate whether actual output exactly matches expected output.
+
+        Args:
+            agent_execution: The execution details containing:
+                - agent_input: The input received by the agent
+                - actual_output: The actual output from the agent
+                - spans: The execution spans to use for the evaluation
+            evaluation_criteria: The criteria to evaluate
+
+        Returns:
+            EvaluationResult: Boolean result indicating exact match (True/False)
+        """
+        return BooleanEvaluationResult(
+            score=self._canonical_json(agent_execution.agent_output)
+            == self._canonical_json(evaluation_criteria)
+        )

uipath/{_cli/_evals/_evaluators/_json_similarity_evaluator.py → eval/evaluators/json_similarity_evaluator.py}

@@ -1,16 +1,18 @@
-import copy
+"""JSON similarity evaluator for flexible structural comparison of outputs."""
+
 import math
-from typing import Any, Dict, Tuple
+from typing import Any, Tuple, TypeVar
+
+from uipath.eval.models import EvaluationResult, NumericEvaluationResult
+
+from ..models.models import AgentExecution
+from .deterministic_evaluator_base import DeterministicEvaluatorBase
 
-from uipath._cli._evals._evaluators._deterministic_evaluator_base import (
-    DeterministicEvaluatorBase,
-)
-from uipath._cli._evals._models import EvaluationResult
-from uipath._cli._evals._models._evaluators import ScoreType
+T = TypeVar("T")
 
 
-class JsonSimilarityEvaluator(DeterministicEvaluatorBase):
-    """Deterministic evaluator that scores structural JSON similarity.
+class JsonSimilarityEvaluator(DeterministicEvaluatorBase[dict[str, Any]]):
+    """Deterministic evaluator that scores structural JSON similarity between expected and actual output.
 
     Compares expected versus actual JSON-like structures and returns a
     numerical score in the range [0, 100]. The comparison is token-based
@@ -18,43 +20,24 @@ class JsonSimilarityEvaluator(DeterministicEvaluatorBase):
     """
 
     async def evaluate(
-        self,
-        evaluation_id: str,
-        evaluation_name: str,
-        input_data: Dict[str, Any],
-        expected_output: Dict[str, Any],
-        actual_output: Dict[str, Any],
+        self, agent_execution: AgentExecution, evaluation_criteria: dict[str, Any]
     ) -> EvaluationResult:
         """Evaluate similarity between expected and actual JSON outputs.
 
-        Args:
-            evaluation_id: Unique identifier for this evaluation run.
-            evaluation_name: Human friendly evaluation name.
-            input_data: Input payload used to produce the outputs.
-            expected_output: Ground-truth JSON structure.
-            actual_output: Produced JSON structure to compare against the ground truth.
+        Uses token-based comparison with tolerance for numeric differences
+        and Levenshtein distance for string similarity.
+
+            agent_execution: The execution details containing:
+                - agent_input: The input received by the agent
+                - actual_output: The actual output from the agent
+                - spans: The execution spans to use for the evaluation
+            evaluation_criteria: The criteria to evaluate
 
         Returns:
-            EvaluationResult: Structured result with the numerical similarity score.
+            EvaluationResult: Numerical score between 0-100 indicating similarity
         """
-        actual_output_copy = copy.deepcopy(actual_output)
-        expected_output_copy = copy.deepcopy(expected_output)
-
-        actual_output, expected_output = self._select_targets(
-            expected_output, actual_output
-        )
-        similarity = self._compare_json(expected_output, actual_output)
-
-        return EvaluationResult(
-            evaluation_id=evaluation_id,
-            evaluation_name=evaluation_name,
-            evaluator_id=self.id,
-            evaluator_name=self.name,
-            score=similarity,
-            input=input_data,
-            expected_output=expected_output_copy,
-            actual_output=actual_output_copy,
-            score_type=ScoreType.NUMERICAL,
+        return NumericEvaluationResult(
+            score=self._compare_json(evaluation_criteria, agent_execution.agent_output)
         )
 
     def _compare_json(self, expected: Any, actual: Any) -> float:

uipath/eval/evaluators/llm_as_judge_evaluator.py

@@ -0,0 +1,137 @@
+"""LLM-as-a-judge evaluator for subjective quality assessment of agent outputs."""
+
+import json
+from typing import Any, Optional
+
+from pydantic import field_validator
+
+from uipath.eval.models import NumericEvaluationResult
+
+from ..._services import UiPathLlmChatService
+from ..._utils.constants import COMMUNITY_agents_SUFFIX
+from ..models.models import AgentExecution, EvaluationResult, LLMResponse
+from .base_evaluator import BaseEvaluator
+
+
+class LlmAsAJudgeEvaluator(BaseEvaluator[dict[str, Any]]):
+    """Evaluator that uses an LLM to judge the quality of agent output."""
+
+    prompt: str
+    model: str
+    actual_output_placeholder: str = "{{ActualOutput}}"
+    expected_output_placeholder: str = "{{ExpectedOutput}}"
+    llm: Optional[UiPathLlmChatService] = None
+
+    @field_validator("prompt")
+    @classmethod
+    def validate_prompt_placeholders(cls, v: str) -> str:
+        """Validate that prompt contains required placeholders."""
+        if "{{ActualOutput}}" not in v or "{{ExpectedOutput}}" not in v:
+            raise ValueError(
+                "Prompt must contain both {ActualOutput} and {ExpectedOutput} placeholders"
+            )
+        return v
+
+    def model_post_init(self, __context):
+        """Initialize the LLM service after model creation."""
+        super().model_post_init(__context)
+        self._initialize_llm()
+
+    def _initialize_llm(self):
+        """Initialize the LLM used for evaluation."""
+        from uipath import UiPath
+
+        uipath = UiPath()
+        self.llm = uipath.llm
+
+    async def evaluate(
+        self,
+        agent_execution: AgentExecution,
+        evaluation_criteria: dict[str, Any],
+    ) -> EvaluationResult:
+        """Evaluate using an LLM as a judge.
+
+        Sends the formatted prompt to the configured LLM and expects a JSON response
+        with a numerical score (0-100) and justification.
+
+            agent_execution: The execution details containing:
+                - agent_input: The input received by the agent
+                - actual_output: The actual output from the agent
+                - spans: The execution spans to use for the evaluation
+            evaluation_criteria: The criteria to evaluate
+
+        Returns:
+            EvaluationResult: Numerical score with LLM justification as details
+        """
+        # Create the evaluation prompt
+        evaluation_prompt = self._create_evaluation_prompt(
+            expected_output=evaluation_criteria,
+            actual_output=agent_execution.agent_output,
+        )
+
+        llm_response = await self._get_llm_response(evaluation_prompt)
+
+        return NumericEvaluationResult(
+            score=llm_response.score,
+            details=llm_response.justification,
+        )
+
+    def _create_evaluation_prompt(
+        self, expected_output: Any, actual_output: Any
+    ) -> str:
+        """Create the evaluation prompt for the LLM."""
+        formatted_prompt = self.prompt.replace(
+            self.actual_output_placeholder,
+            str(actual_output),
+        )
+        formatted_prompt = formatted_prompt.replace(
+            self.expected_output_placeholder,
+            str(expected_output),
+        )
+
+        return formatted_prompt
+
+    async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse:
+        """Get response from the LLM.
+
+        Args:
+            evaluation_prompt: The formatted prompt to send to the LLM
+
+        Returns:
+            LLMResponse with score and justification
+        """
+        # remove community-agents suffix from llm model name
+        model = self.model
+        if model.endswith(COMMUNITY_agents_SUFFIX):
+            model = model.replace(COMMUNITY_agents_SUFFIX, "")
+
+        # Prepare the request
+        request_data = {
+            "model": model,
+            "messages": [{"role": "user", "content": evaluation_prompt}],
+            "response_format": {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": "evaluation_response",
+                    "schema": {
+                        "type": "object",
+                        "properties": {
+                            "score": {
+                                "type": "number",
+                                "minimum": 0,
+                                "maximum": 100,
+                                "description": "Score between 0 and 100",
+                            },
+                            "justification": {
+                                "type": "string",
+                                "description": "Explanation for the score",
+                            },
+                        },
+                        "required": ["score", "justification"],
+                    },
+                },
+            },
+        }
+
+        response = await self.llm.chat_completions(**request_data)  # type: ignore
+        return LLMResponse(**json.loads(response.choices[-1].message.content))

uipath/eval/evaluators/trajectory_evaluator.py

@@ -0,0 +1,36 @@
+"""Trajectory evaluator for analyzing execution paths and decision sequences."""
+
+from typing import TypeVar
+
+from uipath.eval.models import EvaluationResult
+
+from ..models.models import AgentExecution
+from .base_evaluator import BaseEvaluator
+
+T = TypeVar("T")
+
+
+class TrajectoryEvaluator(BaseEvaluator[T]):
+    """Evaluator that analyzes the trajectory/path taken to reach outputs."""
+
+    async def evaluate(
+        self, agent_execution: AgentExecution, evaluation_criteria: T
+    ) -> EvaluationResult:
+        """Evaluate using trajectory analysis.
+
+        Analyzes the execution path and decision sequence taken by the agent
+        to assess the quality of the reasoning process.
+
+        Args:
+            agent_execution: The execution details containing:
+                - agent_input: The input received by the agent
+                - actual_output: The actual output from the agent
+                - spans: The execution spans to use for the evaluation
+            evaluation_criteria: The criteria to evaluate
+        Returns:
+            EvaluationResult: Score based on trajectory analysis
+
+        Raises:
+            NotImplementedError: This evaluator is not yet implemented
+        """
+        raise NotImplementedError()

uipath/eval/models/__init__.py

@@ -0,0 +1,19 @@
+"""UiPath evaluation module for agent performance assessment."""
+
+from uipath.eval.models.models import (
+    BooleanEvaluationResult,
+    ErrorEvaluationResult,
+    EvalItemResult,
+    EvaluationResult,
+    NumericEvaluationResult,
+    ScoreType,
+)
+
+__all__ = [
+    "EvaluationResult",
+    "ScoreType",
+    "EvalItemResult",
+    "BooleanEvaluationResult",
+    "NumericEvaluationResult",
+    "ErrorEvaluationResult",
+]

uipath/{_cli/_evals/_models/_evaluators.py → eval/models/models.py}

@@ -1,15 +1,79 @@
-from datetime import datetime, timezone
+"""Models for evaluation framework including execution data and evaluation results."""
+
 from enum import IntEnum
-from typing import Any, Dict, List, Optional
+from typing import Annotated, Any, Dict, Literal, Optional, Union
+
+from opentelemetry.sdk.trace import ReadableSpan
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class AgentExecution(BaseModel):
+    """Represents the execution data of an agent for evaluation purposes."""
 
-from pydantic import BaseModel
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    agent_input: Optional[Dict[str, Any]]
+    agent_output: Dict[str, Any]
+    agent_trace: list[ReadableSpan]
 
 
 class LLMResponse(BaseModel):
+    """Response from an LLM evaluator."""
+
     score: float
     justification: str
 
 
+class ScoreType(IntEnum):
+    """Types of evaluation scores."""
+
+    BOOLEAN = 0
+    NUMERICAL = 1
+    ERROR = 2
+
+
+class BaseEvaluationResult(BaseModel):
+    """Base class for evaluation results."""
+
+    details: Optional[str] = None
+    # this is marked as optional, as it is populated inside the 'measure_execution_time' decorator
+    evaluation_time: Optional[float] = None
+
+
+class BooleanEvaluationResult(BaseEvaluationResult):
+    """Result of a boolean evaluation."""
+
+    score: bool
+    score_type: Literal[ScoreType.BOOLEAN] = ScoreType.BOOLEAN
+
+
+class NumericEvaluationResult(BaseEvaluationResult):
+    """Result of a numerical evaluation."""
+
+    score: float
+    score_type: Literal[ScoreType.NUMERICAL] = ScoreType.NUMERICAL
+
+
+class ErrorEvaluationResult(BaseEvaluationResult):
+    """Result of an error evaluation."""
+
+    score: float = 0.0
+    score_type: Literal[ScoreType.ERROR] = ScoreType.ERROR
+
+
+EvaluationResult = Annotated[
+    Union[BooleanEvaluationResult, NumericEvaluationResult, ErrorEvaluationResult],
+    Field(discriminator="score_type"),
+]
+
+
+class EvalItemResult(BaseModel):
+    """Result of a single evaluation item."""
+
+    evaluator_name: str
+    result: EvaluationResult
+
+
 class EvaluatorCategory(IntEnum):
     """Types of evaluators."""
 
@@ -48,43 +112,3 @@ class EvaluatorType(IntEnum):
             return cls(value)
         else:
             raise ValueError(f"{value} is not a valid EvaluatorType value")
-
-
-class ScoreType(IntEnum):
-    BOOLEAN = 0
-    NUMERICAL = 1
-    ERROR = 2
-
-
-class EvaluationResult(BaseModel):
-    """Result of a single evaluation."""
-
-    evaluation_id: str
-    evaluation_name: str
-    evaluator_id: str
-    evaluator_name: str
-    score: float | bool
-    score_type: ScoreType
-    # this is marked as optional, as it is populated inside the 'measure_execution_time' decorator
-    evaluation_time: Optional[float] = None
-    input: Dict[str, Any]
-    expected_output: Dict[str, Any]
-    actual_output: Dict[str, Any]
-    timestamp: datetime = datetime.now(timezone.utc)
-    details: Optional[str] = None
-
-
-class EvaluationSetResult(BaseModel):
-    """Result of a complete evaluation set."""
-
-    eval_set_id: str
-    eval_set_name: str
-    results: List[EvaluationResult]
-    average_score: float
-
-
-class EvalItemResult(BaseModel):
-    """Result of a single evaluation item."""
-
-    evaluator_id: str
-    result: EvaluationResult

{uipath-2.1.52.dist-info → uipath-2.1.53.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: uipath
-Version: 2.1.52
+Version: 2.1.53
 Summary: Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools.
 Project-URL: Homepage, https://uipath.com
 Project-URL: Repository, https://github.com/UiPath/uipath-python

{uipath-2.1.52.dist-info → uipath-2.1.53.dist-info}/RECORD

@@ -9,7 +9,7 @@ uipath/_cli/__init__.py,sha256=tscKceSouYcEOxUbGjoyHi4qGi74giBFeXG1I-ut1hs,2308
 uipath/_cli/cli_auth.py,sha256=i3ykLlCg68xgPXHHaa0agHwGFIiLiTLzOiF6Su8XaEo,2436
 uipath/_cli/cli_deploy.py,sha256=KPCmQ0c_NYD5JofSDao5r6QYxHshVCRxlWDVnQvlp5w,645
 uipath/_cli/cli_dev.py,sha256=nEfpjw1PZ72O6jmufYWVrueVwihFxDPOeJakdvNHdOA,2146
-uipath/_cli/cli_eval.py,sha256=fYJWQlyiIc8SpTzY9QPNQWOx40PagMEKdsGZIu9As2A,4402
+uipath/_cli/cli_eval.py,sha256=uiisQ3Wpalu8e9gHRMmn_9Gqus_t4brbjTegMQOhLa0,3831
 uipath/_cli/cli_init.py,sha256=Ac3-9tIH3rpikIX1ehWTo7InW5tjVNoz_w6fjvgLK4w,7052
 uipath/_cli/cli_invoke.py,sha256=4jyhqcy7tPrpxvaUhW-9gut6ddsCGMdJJcpOXXmIe8g,4348
 uipath/_cli/cli_new.py,sha256=9378NYUBc9j-qKVXV7oja-jahfJhXBg8zKVyaon7ctY,2102
@@ -44,22 +44,13 @@ uipath/_cli/_dev/_terminal/_styles/terminal.tcss,sha256=ktVpKwXIXw2VZp8KIZD6fO9i
 uipath/_cli/_dev/_terminal/_utils/_chat.py,sha256=YUZxYVdmEManwHDuZsczJT1dWIYE1dVBgABlurwMFcE,8493
 uipath/_cli/_dev/_terminal/_utils/_exporter.py,sha256=oI6D_eMwrh_2aqDYUh4GrJg8VLGrLYhDahR-_o0uJns,4144
 uipath/_cli/_dev/_terminal/_utils/_logger.py,sha256=jeNShEED27cNIHTe_NNx-2kUiXpSLTmi0onM6tVkqRM,888
-uipath/_cli/_evals/_runtime.py,sha256=q4h3zp_7Ygkhj1zE_YTKKXRp3BhkHaPj8CWqjkzerTk,4748
-uipath/_cli/_evals/progress_reporter.py,sha256=PGt1rs7IH1C6HPw8fWUwb98GB3UBuM6eUiiqGthfCIk,11174
-uipath/_cli/_evals/_evaluators/__init__.py,sha256=jD7KNLjbsUpsESFXX11eW2MEPXDNuPp2-t-IPB-inlM,734
-uipath/_cli/_evals/_evaluators/_deterministic_evaluator_base.py,sha256=BTl0puBjp9iCsU3YFfYWqk4TOz4iE19O3q1-dK6qUOI,1723
-uipath/_cli/_evals/_evaluators/_evaluator_base.py,sha256=knHUwYFt0gMG1uJhq5TGEab6M_YevxX019yT3yYwZsw,3787
-uipath/_cli/_evals/_evaluators/_evaluator_factory.py,sha256=cURShn17X6BW-_G3rknJXWtlgpeh5UdioLUV6oGCGAU,4912
-uipath/_cli/_evals/_evaluators/_exact_match_evaluator.py,sha256=lvEtAitrZy9myoZLMXLqlBWBPX06Msu67kuFMGSbikM,1319
-uipath/_cli/_evals/_evaluators/_json_similarity_evaluator.py,sha256=HpmkvuwU4Az3IIqFVLUmDvzkqb21pFMxY0sg2biZOMM,7093
-uipath/_cli/_evals/_evaluators/_llm_as_judge_evaluator.py,sha256=nSLZ29xWqALEI53ifr79JPXjyx0T4sr7p-4NygwgAio,6594
-uipath/_cli/_evals/_evaluators/_trajectory_evaluator.py,sha256=dnogQTOskpI4_cNF0Ge3hBceJJocvOgxBWAwaCWnzB0,1595
-uipath/_cli/_evals/_models/__init__.py,sha256=Ewjp3u2YeTH2MmzY9LWf7EIbAoIf_nW9fMYbj7pGlPs,420
-uipath/_cli/_evals/_models/_agent_execution_output.py,sha256=llvApU4JkTnNgQ5DvHPt8ee3bnV6cCANyeiebWKE07E,401
-uipath/_cli/_evals/_models/_evaluation_set.py,sha256=tVHykSget-G3sOCs9bSchMYUTpFqzXVlYYbY8L9SI0c,1518
-uipath/_cli/_evals/_models/_evaluators.py,sha256=l57NEVyYmzSKuoIXuGkE94Br01hAMg35fiS2MlTkaQM,2115
+uipath/_cli/_evals/_evaluator_factory.py,sha256=2lOalabNSzmnnwr0SfoPWvFWXs0Ly857XBmPuOdhFBQ,4729
+uipath/_cli/_evals/_runtime.py,sha256=KFGl2we1RH0omuD2HWw5thIK6DDZxVGtqx_G9T4DM_A,8332
+uipath/_cli/_evals/_models/_evaluation_set.py,sha256=mwcTstHuyHd7ys_nLzgCNKBAsS4ns9UL2TF5Oq2Cc64,1758
+uipath/_cli/_evals/_models/_evaluator_base_params.py,sha256=lTYKOV66tcjW85KHTyOdtF1p1VDaBNemrMAvH8bFIFc,382
+uipath/_cli/_evals/_models/_output.py,sha256=TTQ0hhmD3dTkIbj_Ly_rDCGSnpZsHwdmCsl7FLdoZD0,2634
 uipath/_cli/_push/sw_file_handler.py,sha256=AX4TKM-q6CNGw3JyBW02M8ktPZuFMcAU9LN3Ii0Q2QI,18202
-uipath/_cli/_runtime/_contracts.py,sha256=Mwdb11OULGg9xGSu_A6KCl7A5jw-fKe5tBiv62MZNVM,28424
+uipath/_cli/_runtime/_contracts.py,sha256=FziI4E1GF-brOwkYSGEUvRAKhLJUQ_2ZwUMefynl944,28764
 uipath/_cli/_runtime/_escalation.py,sha256=x3vI98qsfRA-fL_tNkRVTFXioM5Gv2w0GFcXJJ5eQtg,7981
 uipath/_cli/_runtime/_hitl.py,sha256=VKbM021nVg1HEDnTfucSLJ0LsDn83CKyUtVzofS2qTU,11369
 uipath/_cli/_runtime/_logging.py,sha256=MGklGKPjYKjs7J5Jy9eplA9zCDsdtEbkZdCbTwgut_4,8311
@@ -74,7 +65,7 @@ uipath/_cli/_utils/_common.py,sha256=CzhhkIRfCuQ1-5HLDtjzOyt8KFs1jm6wzrBeU_v2B7c
 uipath/_cli/_utils/_console.py,sha256=scvnrrFoFX6CE451K-PXKV7UN0DUkInbOtDZ5jAdPP0,10070
 uipath/_cli/_utils/_constants.py,sha256=rS8lQ5Nzull8ytajK6lBsz398qiCp1REoAwlHtyBwF0,1415
 uipath/_cli/_utils/_debug.py,sha256=zamzIR4VgbdKADAE4gbmjxDsbgF7wvdr7C5Dqp744Oc,1739
-uipath/_cli/_utils/_eval_set.py,sha256=z0sTEj4lGkLZXfj9vUpMwFPL6LNMs1MSCZ43Efzoc6A,2750
+uipath/_cli/_utils/_eval_set.py,sha256=4aP8yAC-jMrNYaC62Yj8fHD2hNlotGwy63bciQrpdc4,2766
 uipath/_cli/_utils/_folders.py,sha256=UVJcKPfPAVR5HF4AP6EXdlNVcfEF1v5pwGCpoAgBY34,1155
 uipath/_cli/_utils/_input_args.py,sha256=3LGNqVpJItvof75VGm-ZNTUMUH9-c7-YgleM5b2YgRg,5088
 uipath/_cli/_utils/_parse_ast.py,sha256=8Iohz58s6bYQ7rgWtOTjrEInLJ-ETikmOMZzZdIY2Co,20072
@@ -122,6 +113,15 @@ uipath/agent/conversation/meta.py,sha256=3t0eS9UHoAPHre97QTUeVbjDhnMX4zj4-qG6ju0
 uipath/agent/conversation/tool.py,sha256=ol8XI8AVd-QNn5auXNBPcCzOkh9PPFtL7hTK3kqInkU,2191
 uipath/eval/_helpers/__init__.py,sha256=GSmZMryjuO3Wo_zdxZdrHCRRsgOxsVFYkYgJ15YNC3E,86
 uipath/eval/_helpers/helpers.py,sha256=iE2HHdMiAdAMLqxHkPKHpfecEtAuN5BTBqvKFTI8ciE,1315
+uipath/eval/evaluators/__init__.py,sha256=DJAAhgv0I5UfBod4sGnSiKerfrz1iMmk7GNFb71V8eI,494
+uipath/eval/evaluators/base_evaluator.py,sha256=gryaN7WMV__NGorwu4WPRL5A5RlJ1exQ9jDJ6ZrXDB8,2679
+uipath/eval/evaluators/deterministic_evaluator_base.py,sha256=yDWTMU1mG-93D6DscAUHmaVUc1rhGYtNjGXgevzAObM,1723
+uipath/eval/evaluators/exact_match_evaluator.py,sha256=Qfz-kIUf80PKjAuge1Tc1GvN6kDB6hHveBZ86w_2How,1512
+uipath/eval/evaluators/json_similarity_evaluator.py,sha256=cP4kpN-UIf690V5dq4LaCjJc2zFx-nEffUclCwDdlhM,6607
+uipath/eval/evaluators/llm_as_judge_evaluator.py,sha256=l0bbn8ZLi9ZTXcgr7tJ2tsCvHFqIIeGa7sobaAHgI2Y,4927
+uipath/eval/evaluators/trajectory_evaluator.py,sha256=7boiKzjLpQPs8M8y2PGnI3bZQ1MEwR6QRZpXyKQcR7Y,1244
+uipath/eval/models/__init__.py,sha256=x360CDZaRjUL3q3kh2CcXYYrQ47jwn6p6JnmhEIvMlA,419
+uipath/eval/models/models.py,sha256=9IraD5C2KfKK1ZLMZ7jBOJzzHW4X1Dp2k41abqmPMnA,2838
 uipath/models/__init__.py,sha256=d_DkK1AtRUetM1t2NrH5UKgvJOBiynzaKnK5pMY7aIc,1289
 uipath/models/action_schema.py,sha256=tBn1qQ3NQLU5nwWlBIzIKIx3XK5pO_D1S51IjFlZ1FA,610
 uipath/models/actions.py,sha256=1vRsJ3JSmMdPkbiYAiHzY8K44vmW3VlMsmQUBAkSgrQ,3141
@@ -148,8 +148,8 @@ uipath/tracing/_traced.py,sha256=qeVDrds2OUnpdUIA0RhtF0kg2dlAZhyC1RRkI-qivTM,185
 uipath/tracing/_utils.py,sha256=wJRELaPu69iY0AhV432Dk5QYf_N_ViRU4kAUG1BI1ew,10384
 uipath/utils/__init__.py,sha256=VD-KXFpF_oWexFg6zyiWMkxl2HM4hYJMIUDZ1UEtGx0,105
 uipath/utils/_endpoints_manager.py,sha256=iRTl5Q0XAm_YgcnMcJOXtj-8052sr6jpWuPNz6CgT0Q,8408
-uipath-2.1.52.dist-info/METADATA,sha256=zNHWskIn1OPB0hrLbZBoX6qmd-U-52O_LTHBZ06FvdQ,6482
-uipath-2.1.52.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-uipath-2.1.52.dist-info/entry_points.txt,sha256=9C2_29U6Oq1ExFu7usihR-dnfIVNSKc-0EFbh0rskB4,43
-uipath-2.1.52.dist-info/licenses/LICENSE,sha256=-KBavWXepyDjimmzH5fVAsi-6jNVpIKFc2kZs0Ri4ng,1058
-uipath-2.1.52.dist-info/RECORD,,
+uipath-2.1.53.dist-info/METADATA,sha256=QYSNjKWwpoLp7Vuc_HWzFplKDGJu_40CGdIhmf9SxFE,6482
+uipath-2.1.53.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+uipath-2.1.53.dist-info/entry_points.txt,sha256=9C2_29U6Oq1ExFu7usihR-dnfIVNSKc-0EFbh0rskB4,43
+uipath-2.1.53.dist-info/licenses/LICENSE,sha256=-KBavWXepyDjimmzH5fVAsi-6jNVpIKFc2kZs0Ri4ng,1058
+uipath-2.1.53.dist-info/RECORD,,