uipath 2.1.79__tar.gz → 2.1.80__tar.gz

{uipath-2.1.79 → uipath-2.1.80}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: uipath
-Version: 2.1.79
+Version: 2.1.80
 Summary: Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools.
 Project-URL: Homepage, https://uipath.com
 Project-URL: Repository, https://github.com/UiPath/uipath-python

{uipath-2.1.79 → uipath-2.1.80}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.1.79"
+version = "2.1.80"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.10"

{uipath-2.1.79 → uipath-2.1.80}/samples/calculator/main.py

@@ -35,7 +35,7 @@ GET_RANDOM_OPERATOR_EXAMPLES = [ExampleCall(id="example", input="{}", output="{\
 
 @traced()
 @mockable(example_calls=GET_RANDOM_OPERATOR_EXAMPLES)
-def get_random_operator() -> Wrapper:
+async def get_random_operator() -> Wrapper:
     """Get a random operator."""
     return Wrapper(result=random.choice([Operator.ADD, Operator.SUBTRACT, Operator.MULTIPLY, Operator.DIVIDE]))
 
@@ -43,7 +43,7 @@ def get_random_operator() -> Wrapper:
 @traced()
 async def main(input: CalculatorInput) -> CalculatorOutput:
     if input.operator == Operator.RANDOM:
-        operator = get_random_operator().result
+        operator = (await get_random_operator()).result
     else:
         operator = input.operator
     match operator:

{uipath-2.1.79 → uipath-2.1.80}/src/uipath/_cli/_evals/_models/_output.py

@@ -58,43 +58,46 @@ class EvaluationRunResultDto(BaseModel):
     model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
 
     evaluator_name: str
+    evaluator_id: str
     result: EvaluationResultDto
 
+    @model_serializer(mode="wrap")
+    def serialize_model(self, serializer, info):
+        data = serializer(self)
+        if isinstance(data, dict):
+            data.pop("evaluatorId", None)
+        return data
+
 
 class EvaluationRunResult(BaseModel):
     model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
 
-    score: float = 0.0
     evaluation_name: str
     evaluation_run_results: List[EvaluationRunResultDto]
 
-    def compute_average_score(self) -> None:
+    @property
+    def score(self) -> float:
         """Compute average score for this single eval_item."""
         if not self.evaluation_run_results:
-            self.score = 0.0
-            return
+            return 0.0
 
         total_score = sum(dto.result.score for dto in self.evaluation_run_results)
-        self.score = total_score / len(self.evaluation_run_results)
+        return total_score / len(self.evaluation_run_results)
 
 
 class UiPathEvalOutput(BaseModel):
     model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
 
     evaluation_set_name: str
-    score: float
     evaluation_set_results: List[EvaluationRunResult]
 
-    def compute_average_score(self) -> None:
-        """Compute overall average by calling eval_item.compute_average_score()."""
+    @property
+    def score(self) -> float:
+        """Compute overall average score from evaluation results."""
         if not self.evaluation_set_results:
-            self.score = 0.0
-            return
-
-        for eval_result in self.evaluation_set_results:
-            eval_result.compute_average_score()
+            return 0.0
 
         eval_item_scores = [
             eval_result.score for eval_result in self.evaluation_set_results
         ]
-        self.score = sum(eval_item_scores) / len(eval_item_scores)
+        return sum(eval_item_scores) / len(eval_item_scores)

{uipath-2.1.79 → uipath-2.1.80}/src/uipath/_cli/_evals/_runtime.py

@@ -1,3 +1,4 @@
+import asyncio
 import json
 import logging
 import uuid
@@ -151,9 +152,6 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
         )
         evaluators = self._load_evaluators(evaluation_set)
 
-        evaluator_averages = {evaluator.id: 0.0 for evaluator in evaluators}
-        evaluator_counts = {evaluator.id: 0 for evaluator in evaluators}
-
         await event_bus.publish(
             EvaluationEvents.CREATE_EVAL_SET_RUN,
             EvalSetRunCreatedEvent(
@@ -165,110 +163,37 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
             ),
         )
 
-        results = UiPathEvalOutput(
-            evaluation_set_name=evaluation_set.name, score=0, evaluation_set_results=[]
-        )
-        for eval_item in evaluation_set.evaluations:
-            set_evaluation_item(eval_item)
-            await event_bus.publish(
-                EvaluationEvents.CREATE_EVAL_RUN,
-                EvalRunCreatedEvent(
-                    execution_id=self.execution_id,
-                    eval_item=eval_item,
-                ),
+        # Check if parallel execution should be used
+        if (
+            self.context.workers
+            and self.context.workers > 1
+            and len(evaluation_set.evaluations) > 1
+        ):
+            eval_run_result_list = await self._execute_parallel(
+                evaluation_set, evaluators, event_bus, self.context.workers
             )
-
-            evaluation_run_results = EvaluationRunResult(
-                evaluation_name=eval_item.name, evaluation_run_results=[]
+        else:
+            eval_run_result_list = await self._execute_sequential(
+                evaluation_set, evaluators, event_bus
             )
+        results = UiPathEvalOutput(
+            evaluation_set_name=evaluation_set.name,
+            evaluation_set_results=eval_run_result_list,
+        )
 
-            results.evaluation_set_results.append(evaluation_run_results)
-
-            try:
-                agent_execution_output = await self.execute_runtime(eval_item)
-                evaluation_item_results: list[EvalItemResult] = []
-
-                for evaluator in evaluators:
-                    evaluation_result = await self.run_evaluator(
-                        evaluator=evaluator,
-                        execution_output=agent_execution_output,
-                        eval_item=eval_item,
-                    )
-
-                    dto_result = EvaluationResultDto.from_evaluation_result(
-                        evaluation_result
-                    )
-                    evaluator_counts[evaluator.id] += 1
-                    count = evaluator_counts[evaluator.id]
-                    evaluator_averages[evaluator.id] += (
-                        dto_result.score - evaluator_averages[evaluator.id]
-                    ) / count
-
-                    evaluation_run_results.evaluation_run_results.append(
-                        EvaluationRunResultDto(
-                            evaluator_name=evaluator.name,
-                            result=dto_result,
-                        )
-                    )
-                    evaluation_item_results.append(
-                        EvalItemResult(
-                            evaluator_id=evaluator.id,
-                            result=evaluation_result,
-                        )
-                    )
-
-                evaluation_run_results.compute_average_score()
-
-                await event_bus.publish(
-                    EvaluationEvents.UPDATE_EVAL_RUN,
-                    EvalRunUpdatedEvent(
-                        execution_id=self.execution_id,
-                        eval_item=eval_item,
-                        eval_results=evaluation_item_results,
-                        success=not agent_execution_output.result.error,
-                        agent_output=agent_execution_output.result.output,
-                        agent_execution_time=agent_execution_output.execution_time,
-                        spans=agent_execution_output.spans,
-                        logs=agent_execution_output.logs,
-                    ),
-                    wait_for_completion=False,
-                )
-            except Exception as e:
-                exception_details = EvalItemExceptionDetails(exception=e)
+        # Computing evaluator averages
+        evaluator_averages: Dict[str, float] = defaultdict(float)
+        evaluator_count: Dict[str, int] = defaultdict(int)
 
-                for evaluator in evaluators:
-                    evaluator_counts[evaluator.id] += 1
-                    count = evaluator_counts[evaluator.id]
-                    evaluator_averages[evaluator.id] += (
-                        0.0 - evaluator_averages[evaluator.id]
-                    ) / count
+        for eval_run_result in results.evaluation_set_results:
+            for result_dto in eval_run_result.evaluation_run_results:
+                evaluator_averages[result_dto.evaluator_id] += result_dto.result.score
+                evaluator_count[result_dto.evaluator_id] += 1
 
-                eval_run_updated_event = EvalRunUpdatedEvent(
-                    execution_id=self.execution_id,
-                    eval_item=eval_item,
-                    eval_results=[],
-                    success=False,
-                    agent_output={},
-                    agent_execution_time=0.0,
-                    exception_details=exception_details,
-                    spans=[],
-                    logs=[],
-                )
-                if isinstance(e, EvaluationRuntimeException):
-                    eval_run_updated_event.spans = e.spans
-                    eval_run_updated_event.logs = e.logs
-                    eval_run_updated_event.exception_details.exception = (  # type: ignore
-                        e.root_exception
-                    )
-                    eval_run_updated_event.exception_details.runtime_exception = True  # type: ignore
-
-                await event_bus.publish(
-                    EvaluationEvents.UPDATE_EVAL_RUN,
-                    eval_run_updated_event,
-                    wait_for_completion=False,
-                )
-
-        results.compute_average_score()
+        for eval_id in evaluator_averages:
+            evaluator_averages[eval_id] = (
+                evaluator_averages[eval_id] / evaluator_count[eval_id]
+            )
 
         await event_bus.publish(
             EvaluationEvents.UPDATE_EVAL_SET_RUN,
@@ -285,6 +210,182 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
         )
         return self.context.result
 
+    async def _execute_sequential(
+        self,
+        evaluation_set: EvaluationSet,
+        evaluators: List[BaseEvaluator[Any]],
+        event_bus: EventBus,
+    ) -> List[EvaluationRunResult]:
+        all_eval_run_result: list[EvaluationRunResult] = []
+
+        for eval_item in evaluation_set.evaluations:
+            all_eval_run_result.append(
+                await self._execute_eval(eval_item, evaluators, event_bus)
+            )
+
+        return all_eval_run_result
+
+    async def _execute_parallel(
+        self,
+        evaluation_set: EvaluationSet,
+        evaluators: List[BaseEvaluator[Any]],
+        event_bus: EventBus,
+        workers: int,
+    ) -> List[EvaluationRunResult]:
+        # Create a queue with max concurrency
+        queue: asyncio.Queue[tuple[int, EvaluationItem]] = asyncio.Queue(
+            maxsize=workers
+        )
+
+        # Dictionary to store results with their original indices
+        results_dict: Dict[int, EvaluationRunResult] = {}
+
+        # Producer task to fill the queue
+        async def producer() -> None:
+            for index, eval_item in enumerate(evaluation_set.evaluations):
+                await queue.put((index, eval_item))
+            # Signal completion by putting None markers
+            for _ in range(workers):
+                await queue.put(None)  # type: ignore
+
+        # Worker function to process items from the queue
+        async def worker(worker_id: int) -> None:
+            while True:
+                item = await queue.get()
+
+                # Check for termination signal
+                if item is None:
+                    queue.task_done()
+                    break
+
+                index, eval_item = item
+
+                try:
+                    # Execute the evaluation
+                    result = await self._execute_eval(eval_item, evaluators, event_bus)
+
+                    # Store result with its index to maintain order
+                    results_dict[index] = result
+                finally:
+                    # Mark the task as done
+                    queue.task_done()
+
+        # Start producer
+        producer_task = asyncio.create_task(producer())
+
+        # Create worker tasks based on workers
+        worker_tasks = [asyncio.create_task(worker(i)) for i in range(workers)]
+
+        # Wait for producer and all workers to complete
+        await producer_task
+        await asyncio.gather(*worker_tasks)
+
+        # Return results in the original order
+        return [results_dict[i] for i in range(len(evaluation_set.evaluations))]
+
+    async def _execute_eval(
+        self,
+        eval_item: EvaluationItem,
+        evaluators: List[BaseEvaluator[Any]],
+        event_bus: EventBus,
+    ) -> EvaluationRunResult:
+        set_evaluation_item(eval_item)
+
+        await event_bus.publish(
+            EvaluationEvents.CREATE_EVAL_RUN,
+            EvalRunCreatedEvent(
+                execution_id=self.execution_id,
+                eval_item=eval_item,
+            ),
+        )
+
+        evaluation_run_results = EvaluationRunResult(
+            evaluation_name=eval_item.name, evaluation_run_results=[]
+        )
+
+        try:
+            agent_execution_output = await self.execute_runtime(eval_item)
+            evaluation_item_results: list[EvalItemResult] = []
+
+            for evaluator in evaluators:
+                evaluation_result = await self.run_evaluator(
+                    evaluator=evaluator,
+                    execution_output=agent_execution_output,
+                    eval_item=eval_item,
+                )
+
+                dto_result = EvaluationResultDto.from_evaluation_result(
+                    evaluation_result
+                )
+
+                evaluation_run_results.evaluation_run_results.append(
+                    EvaluationRunResultDto(
+                        evaluator_name=evaluator.name,
+                        result=dto_result,
+                        evaluator_id=evaluator.id,
+                    )
+                )
+                evaluation_item_results.append(
+                    EvalItemResult(
+                        evaluator_id=evaluator.id,
+                        result=evaluation_result,
+                    )
+                )
+
+            await event_bus.publish(
+                EvaluationEvents.UPDATE_EVAL_RUN,
+                EvalRunUpdatedEvent(
+                    execution_id=self.execution_id,
+                    eval_item=eval_item,
+                    eval_results=evaluation_item_results,
+                    success=not agent_execution_output.result.error,
+                    agent_output=agent_execution_output.result.output,
+                    agent_execution_time=agent_execution_output.execution_time,
+                    spans=agent_execution_output.spans,
+                    logs=agent_execution_output.logs,
+                ),
+                wait_for_completion=False,
+            )
+
+        except Exception as e:
+            exception_details = EvalItemExceptionDetails(exception=e)
+
+            for evaluator in evaluators:
+                evaluation_run_results.evaluation_run_results.append(
+                    EvaluationRunResultDto(
+                        evaluator_name=evaluator.name,
+                        evaluator_id=evaluator.id,
+                        result=EvaluationResultDto(score=0),
+                    )
+                )
+
+            eval_run_updated_event = EvalRunUpdatedEvent(
+                execution_id=self.execution_id,
+                eval_item=eval_item,
+                eval_results=[],
+                success=False,
+                agent_output={},
+                agent_execution_time=0.0,
+                exception_details=exception_details,
+                spans=[],
+                logs=[],
+            )
+            if isinstance(e, EvaluationRuntimeException):
+                eval_run_updated_event.spans = e.spans
+                eval_run_updated_event.logs = e.logs
+                eval_run_updated_event.exception_details.exception = (  # type: ignore
+                    e.root_exception
+                )
+                eval_run_updated_event.exception_details.runtime_exception = True  # type: ignore
+
+            await event_bus.publish(
+                EvaluationEvents.UPDATE_EVAL_RUN,
+                eval_run_updated_event,
+                wait_for_completion=False,
+            )
+
+        return evaluation_run_results
+
     def _get_and_clear_execution_data(
         self, execution_id: str
     ) -> tuple[List[ReadableSpan], list[logging.LogRecord]]:

{uipath-2.1.79 → uipath-2.1.80}/src/uipath/_cli/_runtime/_runtime.py

@@ -45,7 +45,9 @@ class UiPathRuntime(UiPathBaseRuntime):
         try:
             script_result = await self.executor(self.context.input_json)
 
-            if self.context.job_id is None:
+            if self.context.job_id is None and not getattr(
+                self.context, "is_eval_run", False
+            ):
                 logger.info(script_result)
 
             self.context.result = UiPathRuntimeResult(

{uipath-2.1.79 → uipath-2.1.80}/src/uipath/_cli/cli_eval.py

@@ -69,8 +69,8 @@ def setup_reporting_prereq(no_report: bool) -> bool:
 @click.option(
     "--workers",
     type=int,
-    default=8,
-    help="Number of parallel workers for running evaluations (default: 8)",
+    default=1,
+    help="Number of parallel workers for running evaluations (default: 1)",
 )
 @click.option(
     "--output-file",