uipath 2.1.108__py3-none-any.whl → 2.1.109__py3-none-any.whl

uipath/eval/_helpers/helpers.py

@@ -1,10 +1,13 @@
+import functools
 import json
 import os
+import time
+from collections.abc import Callable
+from typing import Any
 
 import click
 
-from uipath._cli._utils._console import ConsoleLogger
-from uipath._utils.constants import UIPATH_CONFIG_FILE
+from ..models import ErrorEvaluationResult, EvaluationResult
 
 
 def auto_discover_entrypoint() -> str:
@@ -16,6 +19,9 @@ def auto_discover_entrypoint() -> str:
     Raises:
         ValueError: If no entrypoint found or multiple entrypoints exist
     """
+    from uipath._cli._utils._console import ConsoleLogger
+    from uipath._utils.constants import UIPATH_CONFIG_FILE
+
     console = ConsoleLogger()
 
     if not os.path.isfile(UIPATH_CONFIG_FILE):
@@ -45,3 +51,25 @@ def auto_discover_entrypoint() -> str:
         f"Auto-discovered agent entrypoint: {click.style(entrypoint, fg='cyan')}"
     )
     return entrypoint
+
+
+def track_evaluation_metrics(func: Callable[..., Any]) -> Callable[..., Any]:
+    """Decorator to track evaluation metrics and handle errors gracefully."""
+
+    @functools.wraps(func)
+    async def wrapper(*args: Any, **kwargs: Any) -> EvaluationResult:
+        start_time = time.time()
+        try:
+            result = await func(*args, **kwargs)
+        except Exception as e:
+            result = ErrorEvaluationResult(
+                details="Exception thrown by evaluator: {}".format(e),
+                evaluation_time=time.time() - start_time,
+            )
+        end_time = time.time()
+        execution_time = end_time - start_time
+
+        result.evaluation_time = execution_time
+        return result
+
+    return wrapper

uipath/eval/evaluators/__init__.py

@@ -1,15 +1,70 @@
 """UiPath evaluator implementations for agent performance evaluation."""
 
-from .base_evaluator import BaseEvaluator
+from typing import Any
+
+# Current coded evaluators
+from .base_evaluator import BaseEvaluationCriteria, BaseEvaluator, BaseEvaluatorConfig
+from .contains_evaluator import ContainsEvaluator
 from .exact_match_evaluator import ExactMatchEvaluator
 from .json_similarity_evaluator import JsonSimilarityEvaluator
-from .llm_as_judge_evaluator import LlmAsAJudgeEvaluator
-from .trajectory_evaluator import TrajectoryEvaluator
+
+# Legacy evaluators
+from .legacy_base_evaluator import LegacyBaseEvaluator
+from .legacy_exact_match_evaluator import LegacyExactMatchEvaluator
+from .legacy_json_similarity_evaluator import LegacyJsonSimilarityEvaluator
+from .legacy_llm_as_judge_evaluator import LegacyLlmAsAJudgeEvaluator
+from .legacy_trajectory_evaluator import LegacyTrajectoryEvaluator
+from .llm_judge_output_evaluator import (
+    BaseLLMOutputEvaluator,
+    LLMJudgeOutputEvaluator,
+    LLMJudgeStrictJSONSimilarityOutputEvaluator,
+)
+from .llm_judge_trajectory_evaluator import (
+    BaseLLMTrajectoryEvaluator,
+    LLMJudgeTrajectoryEvaluator,
+    LLMJudgeTrajectorySimulationEvaluator,
+)
+from .tool_call_args_evaluator import ToolCallArgsEvaluator
+from .tool_call_count_evaluator import ToolCallCountEvaluator
+from .tool_call_order_evaluator import ToolCallOrderEvaluator
+from .tool_call_output_evaluator import ToolCallOutputEvaluator
+
+EVALUATORS: list[type[BaseEvaluator[Any, Any, Any]]] = [
+    ExactMatchEvaluator,
+    ContainsEvaluator,
+    JsonSimilarityEvaluator,
+    LLMJudgeOutputEvaluator,
+    LLMJudgeStrictJSONSimilarityOutputEvaluator,
+    LLMJudgeTrajectoryEvaluator,
+    LLMJudgeTrajectorySimulationEvaluator,
+    ToolCallOrderEvaluator,
+    ToolCallArgsEvaluator,
+    ToolCallCountEvaluator,
+    ToolCallOutputEvaluator,
+]
 
 __all__ = [
+    # Legacy evaluators
+    "LegacyBaseEvaluator",
+    "LegacyExactMatchEvaluator",
+    "LegacyJsonSimilarityEvaluator",
+    "LegacyLlmAsAJudgeEvaluator",
+    "LegacyTrajectoryEvaluator",
+    # Current coded evaluators
     "BaseEvaluator",
+    "ContainsEvaluator",
     "ExactMatchEvaluator",
     "JsonSimilarityEvaluator",
-    "LlmAsAJudgeEvaluator",
-    "TrajectoryEvaluator",
+    "BaseLLMOutputEvaluator",
+    "LLMJudgeOutputEvaluator",
+    "LLMJudgeStrictJSONSimilarityOutputEvaluator",
+    "BaseLLMTrajectoryEvaluator",
+    "LLMJudgeTrajectoryEvaluator",
+    "LLMJudgeTrajectorySimulationEvaluator",
+    "ToolCallOrderEvaluator",
+    "ToolCallArgsEvaluator",
+    "ToolCallCountEvaluator",
+    "ToolCallOutputEvaluator",
+    "BaseEvaluationCriteria",
+    "BaseEvaluatorConfig",
 ]