uipath 2.1.108__py3-none-any.whl → 2.1.109__py3-none-any.whl

uipath/_cli/_evals/_runtime.py

@@ -24,7 +24,7 @@ from ..._events._events import (
     EvalSetRunUpdatedEvent,
     EvaluationEvents,
 )
-from ...eval.evaluators import BaseEvaluator
+from ...eval.evaluators import BaseEvaluator, LegacyBaseEvaluator
 from ...eval.models import EvaluationResult
 from ...eval.models.models import AgentExecution, EvalItemResult
 from .._runtime._contracts import (
@@ -38,7 +38,13 @@ from .._runtime._contracts import (
 from .._runtime._logging import ExecutionLogHandler
 from .._utils._eval_set import EvalHelpers
 from ._evaluator_factory import EvaluatorFactory
-from ._models._evaluation_set import EvaluationItem, EvaluationSet
+from ._models._evaluation_set import (
+    AnyEvaluationItem,
+    AnyEvaluationSet,
+    AnyEvaluator,
+    EvaluationItem,
+    LegacyEvaluationItem,
+)
 from ._models._exceptions import EvaluationRuntimeException
 from ._models._output import (
     EvaluationResultDto,
@@ -182,7 +188,8 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
 
         event_bus = self.event_bus
 
-        evaluation_set = EvalHelpers.load_eval_set(
+        # Load eval set (path is already resolved in cli_eval.py)
+        evaluation_set, _ = EvalHelpers.load_eval_set(
             self.context.eval_set, self.context.eval_ids
         )
         evaluators = self._load_evaluators(evaluation_set)
@@ -215,6 +222,7 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
             evaluation_set_name=evaluation_set.name,
             evaluation_set_results=eval_run_result_list,
         )
+
         # Computing evaluator averages
         evaluator_averages: Dict[str, float] = defaultdict(float)
         evaluator_count: Dict[str, int] = defaultdict(int)
@@ -245,8 +253,8 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
 
     async def _execute_sequential(
         self,
-        evaluation_set: EvaluationSet,
-        evaluators: List[BaseEvaluator[Any]],
+        evaluation_set: AnyEvaluationSet,
+        evaluators: List[AnyEvaluator],
         event_bus: EventBus,
     ) -> List[EvaluationRunResult]:
         all_eval_run_result: list[EvaluationRunResult] = []
@@ -260,13 +268,13 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
 
     async def _execute_parallel(
         self,
-        evaluation_set: EvaluationSet,
-        evaluators: List[BaseEvaluator[Any]],
+        evaluation_set: AnyEvaluationSet,
+        evaluators: List[AnyEvaluator],
         event_bus: EventBus,
         workers: int,
     ) -> List[EvaluationRunResult]:
         # Create a queue with max concurrency
-        queue: asyncio.Queue[tuple[int, EvaluationItem]] = asyncio.Queue(
+        queue: asyncio.Queue[tuple[int, AnyEvaluationItem]] = asyncio.Queue(
             maxsize=workers
         )
 
@@ -276,7 +284,7 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
         # Producer task to fill the queue
         async def producer() -> None:
             for index, eval_item in enumerate(evaluation_set.evaluations):
-                await queue.put((index, eval_item))
+                await queue.put((index, eval_item))  # type: ignore[arg-type]
             # Signal completion by putting None markers
             for _ in range(workers):
                 await queue.put(None)  # type: ignore
@@ -318,15 +326,12 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
 
     async def _execute_eval(
         self,
-        eval_item: EvaluationItem,
-        evaluators: List[BaseEvaluator[Any]],
+        eval_item: AnyEvaluationItem,
+        evaluators: List[AnyEvaluator],
         event_bus: EventBus,
     ) -> EvaluationRunResult:
-        # Generate LLM-based input if input_mocking_strategy is defined
-        if eval_item.input_mocking_strategy:
-            eval_item = await self._generate_input_for_eval(eval_item)
-
         execution_id = str(uuid.uuid4())
+
         set_execution_context(eval_item, self.span_collector, execution_id)
 
         await event_bus.publish(
@@ -346,11 +351,41 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
             evaluation_item_results: list[EvalItemResult] = []
 
             for evaluator in evaluators:
-                evaluation_result = await self.run_evaluator(
-                    evaluator=evaluator,
-                    execution_output=agent_execution_output,
-                    eval_item=eval_item,
-                )
+                # Determine which evaluator method to use based on evaluation set/item type
+                evaluation_result: Optional[EvaluationResult] = None
+
+                match eval_item:
+                    case LegacyEvaluationItem():
+                        # Legacy evaluation - use run_legacy_evaluator
+                        evaluation_result = await self.run_legacy_evaluator(
+                            evaluator=evaluator,  # type: ignore
+                            execution_output=agent_execution_output,
+                            eval_item=eval_item,
+                        )
+                    case EvaluationItem() if (
+                        evaluator.id in eval_item.evaluation_criterias
+                    ):
+                        # New evaluation with criteria
+                        evaluation_criteria = eval_item.evaluation_criterias[
+                            evaluator.id
+                        ]
+
+                        evaluation_result = await self.run_evaluator(
+                            evaluator=evaluator,  # type: ignore
+                            execution_output=agent_execution_output,
+                            eval_item=eval_item,
+                            evaluation_criteria=evaluator.evaluation_criteria_type(  # type: ignore
+                                **evaluation_criteria
+                            )
+                            if evaluation_criteria
+                            else evaluator.evaluator_config.default_evaluation_criteria,  # type: ignore
+                        )
+                    case _:
+                        # Skip if evaluator not in evaluation criteria
+                        continue
+
+                if evaluation_result is None:
+                    continue
 
                 dto_result = EvaluationResultDto.from_evaluation_result(
                     evaluation_result
@@ -449,7 +484,7 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
         return spans, logs
 
     async def execute_runtime(
-        self, eval_item: EvaluationItem, execution_id: str
+        self, eval_item: AnyEvaluationItem, execution_id: str
     ) -> UiPathEvalRunExecutionOutput:
         context_args = self.context.model_dump()
         context_args["execution_id"] = execution_id
@@ -486,7 +521,6 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
 
         if result is None:
             raise ValueError("Execution result cannot be None for eval runs")
-
         return UiPathEvalRunExecutionOutput(
             execution_time=end_time - start_time,
             spans=spans,
@@ -501,9 +535,31 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
 
     async def run_evaluator(
         self,
-        evaluator: BaseEvaluator[Any],
+        evaluator: BaseEvaluator[Any, Any, Any],
         execution_output: UiPathEvalRunExecutionOutput,
         eval_item: EvaluationItem,
+        *,
+        evaluation_criteria: Any,
+    ) -> EvaluationResult:
+        agent_execution = AgentExecution(
+            agent_input=eval_item.inputs,
+            agent_output=execution_output.result.output or {},
+            agent_trace=execution_output.spans,
+            expected_agent_behavior=eval_item.expected_agent_behavior,
+        )
+
+        result = await evaluator.validate_and_evaluate_criteria(
+            agent_execution=agent_execution,
+            evaluation_criteria=evaluation_criteria,
+        )
+
+        return result
+
+    async def run_legacy_evaluator(
+        self,
+        evaluator: LegacyBaseEvaluator[Any],
+        execution_output: UiPathEvalRunExecutionOutput,
+        eval_item: LegacyEvaluationItem,
     ) -> EvaluationResult:
         agent_execution = AgentExecution(
             agent_input=eval_item.inputs,
@@ -520,9 +576,7 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
 
         return result
 
-    def _load_evaluators(
-        self, evaluation_set: EvaluationSet
-    ) -> List[BaseEvaluator[Any]]:
+    def _load_evaluators(self, evaluation_set: AnyEvaluationSet) -> list[AnyEvaluator]:
         """Load evaluators referenced by the evaluation set."""
         evaluators = []
         evaluators_dir = Path(self.context.eval_set).parent.parent / "evaluators"  # type: ignore

uipath/_cli/_evals/mocks/input_mocker.py

@@ -67,9 +67,7 @@ async def generate_llm_input(
             if evaluation_item.input_mocking_strategy
             else "",
             expected_behavior=evaluation_item.expected_agent_behavior or "",
-            expected_output=json.dumps(evaluation_item.expected_output, indent=2)
-            if evaluation_item.expected_output
-            else "",
+            expected_output=json.dumps(evaluation_item.evaluation_criterias, indent=2),
         )
 
         response_format = {

uipath/_cli/_evals/mocks/llm_mocker.py

@@ -10,7 +10,7 @@ from uipath.tracing._traced import traced
 from uipath.tracing._utils import _SpanUtils
 
 from .._models._evaluation_set import (
-    EvaluationItem,
+    AnyEvaluationItem,
     LLMMockingStrategy,
 )
 from .._models._mocks import ExampleCall
@@ -77,7 +77,7 @@ def pydantic_to_dict_safe(obj: Any) -> Any:
 class LLMMocker(Mocker):
     """LLM Based Mocker."""
 
-    def __init__(self, evaluation_item: EvaluationItem):
+    def __init__(self, evaluation_item: AnyEvaluationItem):
         """LLM Mocker constructor."""
         self.evaluation_item = evaluation_item
         assert isinstance(self.evaluation_item.mocking_strategy, LLMMockingStrategy)

uipath/_cli/_evals/mocks/mocker_factory.py

@@ -1,7 +1,7 @@
 """Mocker Factory."""
 
 from uipath._cli._evals._models._evaluation_set import (
-    EvaluationItem,
+    AnyEvaluationItem,
     LLMMockingStrategy,
     MockitoMockingStrategy,
 )
@@ -14,7 +14,7 @@ class MockerFactory:
     """Mocker factory."""
 
     @staticmethod
-    def create(evaluation_item: EvaluationItem) -> Mocker:
+    def create(evaluation_item: AnyEvaluationItem) -> Mocker:
         """Create a mocker instance."""
         match evaluation_item.mocking_strategy:
             case LLMMockingStrategy():

uipath/_cli/_evals/mocks/mockito_mocker.py

@@ -9,7 +9,7 @@ from hydra.utils import instantiate
 from mockito import invocation, mocking  # type: ignore[import-untyped]
 
 from uipath._cli._evals._models._evaluation_set import (
-    EvaluationItem,
+    AnyEvaluationItem,
     MockingAnswerType,
     MockitoMockingStrategy,
 )
@@ -38,7 +38,7 @@ class Stub:
 class MockitoMocker(Mocker):
     """Mockito Mocker."""
 
-    def __init__(self, evaluation_item: EvaluationItem):
+    def __init__(self, evaluation_item: AnyEvaluationItem):
         """Instantiate a mockito mocker."""
         self.evaluation_item = evaluation_item
         assert isinstance(self.evaluation_item.mocking_strategy, MockitoMockingStrategy)

uipath/_cli/_evals/mocks/mocks.py

@@ -4,13 +4,13 @@ import logging
 from contextvars import ContextVar
 from typing import Any, Callable, Optional
 
-from uipath._cli._evals._models._evaluation_set import EvaluationItem
+from uipath._cli._evals._models._evaluation_set import AnyEvaluationItem
 from uipath._cli._evals._span_collection import ExecutionSpanCollector
 from uipath._cli._evals.mocks.mocker import Mocker, UiPathNoMockFoundError
 from uipath._cli._evals.mocks.mocker_factory import MockerFactory
 
 # Context variables for evaluation items and mockers
-evaluation_context: ContextVar[Optional[EvaluationItem]] = ContextVar(
+evaluation_context: ContextVar[Optional[AnyEvaluationItem]] = ContextVar(
     "evaluation", default=None
 )
 
@@ -30,7 +30,9 @@ logger = logging.getLogger(__name__)
 
 
 def set_execution_context(
-    eval_item: EvaluationItem, span_collector: ExecutionSpanCollector, execution_id: str
+    eval_item: AnyEvaluationItem,
+    span_collector: ExecutionSpanCollector,
+    execution_id: str,
 ) -> None:
     """Set the execution context for an evaluation run for mocking and trace access."""
     evaluation_context.set(eval_item)

uipath/_cli/_push/models.py

@@ -0,0 +1,17 @@
+"""Models for push command."""
+
+from pydantic import BaseModel, Field
+
+
+class EvaluatorFileDetails(BaseModel):
+    """Details about an evaluator file for push operations."""
+
+    path: str
+    custom_evaluator_file_name: str = Field(
+        "", description="Name of the custom evaluator file, if available."
+    )
+
+    @property
+    def is_custom(self) -> bool:
+        """Check if this is a custom evaluator."""
+        return len(self.custom_evaluator_file_name) > 0