uipath 2.1.16__tar.gz → 2.1.18__tar.gz

{uipath-2.1.16 → uipath-2.1.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: uipath
-Version: 2.1.16
+Version: 2.1.18
 Summary: Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools.
 Project-URL: Homepage, https://uipath.com
 Project-URL: Repository, https://github.com/UiPath/uipath-python

{uipath-2.1.16 → uipath-2.1.18}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.1.16"
+version = "2.1.18"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.10"

{uipath-2.1.16 → uipath-2.1.18}/src/uipath/_cli/_evals/_evaluators/__init__.py

@@ -3,18 +3,20 @@
 This package contains all evaluator types and the factory for creating them.
 """
 
-from ._agent_scorer_evaluator import AgentScorerEvaluator
-from ._deterministic_evaluator import DeterministicEvaluator
+from ._deterministic_evaluator_base import DeterministicEvaluatorBase
 from ._evaluator_base import EvaluatorBase
 from ._evaluator_factory import EvaluatorFactory
+from ._exact_match_evaluator import ExactMatchEvaluator
+from ._json_similarity_evaluator import JsonSimilarityEvaluator
 from ._llm_as_judge_evaluator import LlmAsAJudgeEvaluator
 from ._trajectory_evaluator import TrajectoryEvaluator
 
 __all__ = [
     "EvaluatorBase",
+    "DeterministicEvaluatorBase",
     "EvaluatorFactory",
-    "DeterministicEvaluator",
+    "JsonSimilarityEvaluator",
+    "ExactMatchEvaluator",
     "LlmAsAJudgeEvaluator",
-    "AgentScorerEvaluator",
     "TrajectoryEvaluator",
 ]

uipath-2.1.18/src/uipath/_cli/_evals/_evaluators/_deterministic_evaluator_base.py

@@ -0,0 +1,46 @@
+import copy
+import json
+from abc import ABC
+from typing import Any, Dict, Tuple
+
+from ._evaluator_base import EvaluatorBase
+
+
+class DeterministicEvaluatorBase(EvaluatorBase, ABC):
+    def __init__(self, target_output_key: str = "*"):
+        super().__init__()
+        self.target_output_key = target_output_key
+
+    def _select_targets(
+        self, expected_output: Dict[str, Any], actual_output: Dict[str, Any]
+    ) -> Tuple[Any, Any]:
+        actual_output_copy = copy.deepcopy(actual_output)
+        expected_output_copy = copy.deepcopy(expected_output)
+        if self.target_output_key != "*":
+            if (
+                self.target_output_key not in actual_output
+                or self.target_output_key not in expected_output
+            ):
+                raise ValueError(
+                    f"Field '{self.target_output_key}' missing from expected or actual output"
+                )
+            actual_output_copy = actual_output_copy[self.target_output_key]
+            expected_output_copy = expected_output[self.target_output_key]
+        return actual_output_copy, expected_output_copy
+
+    def _canonical_json(self, obj: Any) -> str:
+        return json.dumps(
+            self._normalize_numbers(obj),
+            sort_keys=True,
+            separators=(",", ":"),
+            ensure_ascii=False,
+        )
+
+    def _normalize_numbers(self, obj: Any) -> Any:
+        if isinstance(obj, dict):
+            return {k: self._normalize_numbers(v) for k, v in obj.items()}
+        if isinstance(obj, (list, tuple)):
+            return [self._normalize_numbers(v) for v in obj]
+        if isinstance(obj, (int, float)) and not isinstance(obj, bool):
+            return float(obj)
+        return obj

{uipath-2.1.16 → uipath-2.1.18}/src/uipath/_cli/_evals/_evaluators/_evaluator_factory.py

@@ -1,9 +1,9 @@
 from typing import Any, Dict
 
 from .._models import EvaluatorCategory, EvaluatorType
-from ._agent_scorer_evaluator import AgentScorerEvaluator
-from ._deterministic_evaluator import DeterministicEvaluator
 from ._evaluator_base import EvaluatorBase, EvaluatorBaseParams
+from ._exact_match_evaluator import ExactMatchEvaluator
+from ._json_similarity_evaluator import JsonSimilarityEvaluator
 from ._llm_as_judge_evaluator import LlmAsAJudgeEvaluator
 from ._trajectory_evaluator import TrajectoryEvaluator
 
@@ -50,23 +50,50 @@ class EvaluatorFactory:
         )
 
         # Create evaluator based on category
-        if category == EvaluatorCategory.Deterministic:
-            return EvaluatorFactory._create_deterministic_evaluator(base_params, data)
-        elif category == EvaluatorCategory.LlmAsAJudge:
-            return EvaluatorFactory._create_llm_as_judge_evaluator(base_params, data)
-        elif category == EvaluatorCategory.AgentScorer:
-            return EvaluatorFactory._create_agent_scorer_evaluator(base_params, data)
-        elif category == EvaluatorCategory.Trajectory:
-            return EvaluatorFactory._create_trajectory_evaluator(base_params, data)
-        else:
-            raise ValueError(f"Unknown evaluator category: {category}")
+        match category:
+            case EvaluatorCategory.Deterministic:
+                if evaluator_type == evaluator_type.Equals:
+                    return EvaluatorFactory._create_exact_match_evaluator(
+                        base_params, data
+                    )
+                elif evaluator_type == evaluator_type.JsonSimilarity:
+                    return EvaluatorFactory._create_json_similarity_evaluator(
+                        base_params, data
+                    )
+                else:
+                    raise ValueError(
+                        f"Unknown evaluator type {evaluator_type} for category {category}"
+                    )
+            case EvaluatorCategory.LlmAsAJudge:
+                return EvaluatorFactory._create_llm_as_judge_evaluator(
+                    base_params, data
+                )
+            case EvaluatorCategory.AgentScorer:
+                raise NotImplementedError()
+            case EvaluatorCategory.Trajectory:
+                return EvaluatorFactory._create_trajectory_evaluator(base_params, data)
+            case _:
+                raise ValueError(f"Unknown evaluator category: {category}")
 
     @staticmethod
-    def _create_deterministic_evaluator(
+    def _create_exact_match_evaluator(
         base_params: EvaluatorBaseParams, data: Dict[str, Any]
-    ) -> DeterministicEvaluator:
+    ) -> ExactMatchEvaluator:
         """Create a deterministic evaluator."""
-        raise NotImplementedError()
+        return ExactMatchEvaluator.from_params(
+            base_params,
+            target_output_key=data.get("targetOutputKey", "*"),
+        )
+
+    @staticmethod
+    def _create_json_similarity_evaluator(
+        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+    ) -> JsonSimilarityEvaluator:
+        """Create a deterministic evaluator."""
+        return JsonSimilarityEvaluator.from_params(
+            base_params,
+            target_output_key=data.get("targetOutputKey", "*"),
+        )
 
     @staticmethod
     def _create_llm_as_judge_evaluator(
@@ -88,13 +115,6 @@ class EvaluatorFactory:
             target_output_key=data.get("targetOutputKey", "*"),
         )
 
-    @staticmethod
-    def _create_agent_scorer_evaluator(
-        base_params: EvaluatorBaseParams, data: Dict[str, Any]
-    ) -> AgentScorerEvaluator:
-        """Create an agent scorer evaluator."""
-        raise NotImplementedError()
-
     @staticmethod
     def _create_trajectory_evaluator(
         base_params: EvaluatorBaseParams, data: Dict[str, Any]

uipath-2.1.18/src/uipath/_cli/_evals/_evaluators/_exact_match_evaluator.py

@@ -0,0 +1,40 @@
+import copy
+from typing import Any, Dict
+
+from uipath._cli._evals._evaluators._deterministic_evaluator_base import (
+    DeterministicEvaluatorBase,
+)
+from uipath._cli._evals._models import EvaluationResult
+from uipath._cli._evals._models._evaluators import ScoreType
+
+
+class ExactMatchEvaluator(DeterministicEvaluatorBase):
+    async def evaluate(
+        self,
+        evaluation_id: str,
+        evaluation_name: str,
+        input_data: Dict[str, Any],
+        expected_output: Dict[str, Any],
+        actual_output: Dict[str, Any],
+    ) -> EvaluationResult:
+        actual_output_copy = copy.deepcopy(actual_output)
+        expected_output_copy = copy.deepcopy(expected_output)
+
+        actual_output, expected_output = self._select_targets(
+            expected_output, actual_output
+        )
+        are_equal = self._canonical_json(actual_output) == self._canonical_json(
+            expected_output
+        )
+
+        return EvaluationResult(
+            evaluation_id=evaluation_id,
+            evaluation_name=evaluation_name,
+            evaluator_id=self.id,
+            evaluator_name=self.name,
+            score=are_equal,
+            input=input_data,
+            expected_output=expected_output_copy,
+            actual_output=actual_output_copy,
+            score_type=ScoreType.BOOLEAN,
+        )

uipath-2.1.18/src/uipath/_cli/_evals/_evaluators/_json_similarity_evaluator.py

@@ -0,0 +1,168 @@
+import copy
+import math
+from typing import Any, Dict, Tuple
+
+from uipath._cli._evals._evaluators._deterministic_evaluator_base import (
+    DeterministicEvaluatorBase,
+)
+from uipath._cli._evals._models import EvaluationResult
+from uipath._cli._evals._models._evaluators import ScoreType
+
+
+class JsonSimilarityEvaluator(DeterministicEvaluatorBase):
+    """Deterministic evaluator that scores structural JSON similarity.
+
+    Compares expected versus actual JSON-like structures and returns a
+    numerical score in the range [0, 100]. The comparison is token-based
+    and tolerant for numbers and strings (via Levenshtein distance).
+    """
+
+    async def evaluate(
+        self,
+        evaluation_id: str,
+        evaluation_name: str,
+        input_data: Dict[str, Any],
+        expected_output: Dict[str, Any],
+        actual_output: Dict[str, Any],
+    ) -> EvaluationResult:
+        """Evaluate similarity between expected and actual JSON outputs.
+
+        Args:
+            evaluation_id: Unique identifier for this evaluation run.
+            evaluation_name: Human friendly evaluation name.
+            input_data: Input payload used to produce the outputs.
+            expected_output: Ground-truth JSON structure.
+            actual_output: Produced JSON structure to compare against the ground truth.
+
+        Returns:
+            EvaluationResult: Structured result with the numerical similarity score.
+        """
+        actual_output_copy = copy.deepcopy(actual_output)
+        expected_output_copy = copy.deepcopy(expected_output)
+
+        actual_output, expected_output = self._select_targets(
+            expected_output, actual_output
+        )
+        similarity = self._compare_json(expected_output, actual_output)
+
+        return EvaluationResult(
+            evaluation_id=evaluation_id,
+            evaluation_name=evaluation_name,
+            evaluator_id=self.id,
+            evaluator_name=self.name,
+            score=similarity,
+            input=input_data,
+            expected_output=expected_output_copy,
+            actual_output=actual_output_copy,
+            score_type=ScoreType.NUMERICAL,
+        )
+
+    def _compare_json(self, expected: Any, actual: Any) -> float:
+        matched_leaves, total_leaves = self._compare_tokens(expected, actual)
+        if total_leaves == 0:
+            return 100.0
+        sim = (matched_leaves / total_leaves) * 100.0
+        return max(0.0, min(100.0, sim))
+
+    def _compare_tokens(
+        self, expected_token: Any, actual_token: Any
+    ) -> Tuple[float, float]:
+        if self._is_number(expected_token) and self._is_number(actual_token):
+            return self._compare_numbers(float(expected_token), float(actual_token))
+
+        if type(expected_token) is not type(actual_token):
+            return 0.0, self._count_leaves(expected_token)
+
+        if isinstance(expected_token, dict):
+            matched_leaves = total_leaves = 0.0
+            # Only expected keys count
+            for expected_key, expected_value in expected_token.items():
+                if isinstance(actual_token, dict) and expected_key in actual_token:
+                    matched, total = self._compare_tokens(
+                        expected_value, actual_token[expected_key]
+                    )
+                else:
+                    matched, total = (0.0, self._count_leaves(expected_value))
+                matched_leaves += matched
+                total_leaves += total
+            return matched_leaves, total_leaves
+
+        if isinstance(expected_token, list):
+            matched_leaves = total_leaves = 0.0
+            common_length = min(len(expected_token), len(actual_token))
+            for index in range(common_length):
+                matched, total = self._compare_tokens(
+                    expected_token[index], actual_token[index]
+                )
+                matched_leaves += matched
+                total_leaves += total
+            for index in range(common_length, len(expected_token)):
+                total_leaves += self._count_leaves(expected_token[index])
+            return (matched_leaves, total_leaves)
+
+        if isinstance(expected_token, bool):
+            return (1.0, 1.0) if expected_token == actual_token else (0.0, 1.0)
+
+        if isinstance(expected_token, str):
+            return self._compare_strings(expected_token, actual_token)
+
+        return (1.0, 1.0) if str(expected_token) == str(actual_token) else (0.0, 1.0)
+
+    def _compare_numbers(
+        self, expected_number: float, actual_number: float
+    ) -> Tuple[float, float]:
+        total = 1.0
+        if math.isclose(expected_number, 0.0, abs_tol=1e-12):
+            matched = 1.0 if math.isclose(actual_number, 0.0, abs_tol=1e-12) else 0.0
+        else:
+            ratio = abs(expected_number - actual_number) / abs(expected_number)
+            matched = max(0.0, min(1.0, 1.0 - ratio))
+        return matched, total
+
+    def _compare_strings(
+        self, expected_string: str, actual_string: str
+    ) -> Tuple[float, float]:
+        total = 1.0
+        if not expected_string and not actual_string:
+            return 1.0, total
+        distance = self._levenshtein(expected_string, actual_string)
+        max_length = max(len(expected_string), len(actual_string))
+        similarity = 1.0 - (distance / max_length) if max_length else 1.0
+        similarity = max(0.0, min(1.0, similarity))
+        return similarity, total
+
+    def _count_leaves(self, token_node: Any) -> float:
+        if isinstance(token_node, dict):
+            return sum(
+                self._count_leaves(child_value) for child_value in token_node.values()
+            )
+        if isinstance(token_node, list):
+            return sum(self._count_leaves(child_value) for child_value in token_node)
+        return 1.0
+
+    def _levenshtein(self, source_text: str, target_text: str) -> int:
+        if not source_text:
+            return len(target_text)
+        if not target_text:
+            return len(source_text)
+        source_len, target_len = len(source_text), len(target_text)
+        distance_matrix = [[0] * (target_len + 1) for _ in range(source_len + 1)]
+        for row_idx in range(source_len + 1):
+            distance_matrix[row_idx][0] = row_idx
+        for col_idx in range(target_len + 1):
+            distance_matrix[0][col_idx] = col_idx
+        for row_idx in range(1, source_len + 1):
+            for col_idx in range(1, target_len + 1):
+                substitution_cost = (
+                    0 if source_text[row_idx - 1] == target_text[col_idx - 1] else 1
+                )
+                distance_matrix[row_idx][col_idx] = min(
+                    distance_matrix[row_idx - 1][col_idx] + 1,  # deletion
+                    distance_matrix[row_idx][col_idx - 1] + 1,  # insertion
+                    distance_matrix[row_idx - 1][col_idx - 1]
+                    + substitution_cost,  # substitution
+                )
+        return distance_matrix[source_len][target_len]
+
+    def _is_number(self, value: Any) -> bool:
+        return isinstance(value, (int, float)) and not isinstance(value, bool)

{uipath-2.1.16 → uipath-2.1.18}/src/uipath/_cli/_evals/_evaluators/_llm_as_judge_evaluator.py

@@ -11,6 +11,7 @@ from ...._utils.constants import (
     COMMUNITY_agents_SUFFIX,
 )
 from .._models import EvaluationResult, LLMResponse
+from .._models._evaluators import ScoreType
 from ._evaluator_base import EvaluatorBase
 
 
@@ -86,6 +87,7 @@ class LlmAsAJudgeEvaluator(EvaluatorBase):
             expected_output=expected_output,
             actual_output=actual_output,
             details=llm_response.justification,
+            score_type=ScoreType.NUMERICAL,
         )
 
     def _extract_target_value(self, output: Dict[str, Any]) -> Any:

{uipath-2.1.16 → uipath-2.1.18}/src/uipath/_cli/_evals/_models/_evaluators.py

@@ -1,8 +1,8 @@
-from datetime import datetime
+from datetime import datetime, timezone
 from enum import IntEnum
 from typing import Any, Dict, List, Optional
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
 
 
 class LLMResponse(BaseModel):
@@ -50,6 +50,12 @@ class EvaluatorType(IntEnum):
             raise ValueError(f"{value} is not a valid EvaluatorType value")
 
 
+class ScoreType(IntEnum):
+    BOOLEAN = 0
+    NUMERICAL = 1
+    ERROR = 2
+
+
 class EvaluationResult(BaseModel):
     """Result of a single evaluation."""
 
@@ -57,13 +63,14 @@ class EvaluationResult(BaseModel):
     evaluation_name: str
     evaluator_id: str
     evaluator_name: str
-    score: float
-    # mark this as optional, as it is populated inside the 'measure_execution_time' decorator
+    score: float | bool
+    score_type: ScoreType
+    # this is marked as optional, as it is populated inside the 'measure_execution_time' decorator
     evaluation_time: Optional[float] = None
     input: Dict[str, Any]
     expected_output: Dict[str, Any]
     actual_output: Dict[str, Any]
-    timestamp: datetime = Field(default_factory=datetime.utcnow)
+    timestamp: datetime = datetime.now(timezone.utc)
     details: Optional[str] = None
 
 
@@ -76,12 +83,6 @@ class EvaluationSetResult(BaseModel):
     average_score: float
 
 
-class ScoreType(IntEnum):
-    BOOLEAN = 0
-    NUMERICAL = 1
-    ERROR = 2
-
-
 class EvalItemResult(BaseModel):
     """Result of a single evaluation item."""
 

{uipath-2.1.16 → uipath-2.1.18}/src/uipath/_cli/_evals/evaluation_service.py

@@ -337,7 +337,7 @@ class EvaluationService:
             try:
                 if self._progress_reporter:
                     await self._progress_reporter.update_eval_run(
-                        eval_results, eval_run_id, success, execution_time
+                        eval_results, eval_run_id, execution_time
                     )
                 sw_progress_reporter_queue.task_done()
             except Exception as e: