uipath 2.1.79__py3-none-any.whl → 2.1.81__py3-none-any.whl

uipath/_cli/_evals/_runtime.py

@@ -1,3 +1,4 @@
+import asyncio
 import json
 import logging
 import uuid
@@ -151,9 +152,6 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
         )
         evaluators = self._load_evaluators(evaluation_set)
 
-        evaluator_averages = {evaluator.id: 0.0 for evaluator in evaluators}
-        evaluator_counts = {evaluator.id: 0 for evaluator in evaluators}
-
         await event_bus.publish(
             EvaluationEvents.CREATE_EVAL_SET_RUN,
             EvalSetRunCreatedEvent(
@@ -165,110 +163,37 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
             ),
         )
 
-        results = UiPathEvalOutput(
-            evaluation_set_name=evaluation_set.name, score=0, evaluation_set_results=[]
-        )
-        for eval_item in evaluation_set.evaluations:
-            set_evaluation_item(eval_item)
-            await event_bus.publish(
-                EvaluationEvents.CREATE_EVAL_RUN,
-                EvalRunCreatedEvent(
-                    execution_id=self.execution_id,
-                    eval_item=eval_item,
-                ),
+        # Check if parallel execution should be used
+        if (
+            self.context.workers
+            and self.context.workers > 1
+            and len(evaluation_set.evaluations) > 1
+        ):
+            eval_run_result_list = await self._execute_parallel(
+                evaluation_set, evaluators, event_bus, self.context.workers
             )
-
-            evaluation_run_results = EvaluationRunResult(
-                evaluation_name=eval_item.name, evaluation_run_results=[]
+        else:
+            eval_run_result_list = await self._execute_sequential(
+                evaluation_set, evaluators, event_bus
             )
+        results = UiPathEvalOutput(
+            evaluation_set_name=evaluation_set.name,
+            evaluation_set_results=eval_run_result_list,
+        )
 
-            results.evaluation_set_results.append(evaluation_run_results)
-
-            try:
-                agent_execution_output = await self.execute_runtime(eval_item)
-                evaluation_item_results: list[EvalItemResult] = []
-
-                for evaluator in evaluators:
-                    evaluation_result = await self.run_evaluator(
-                        evaluator=evaluator,
-                        execution_output=agent_execution_output,
-                        eval_item=eval_item,
-                    )
-
-                    dto_result = EvaluationResultDto.from_evaluation_result(
-                        evaluation_result
-                    )
-                    evaluator_counts[evaluator.id] += 1
-                    count = evaluator_counts[evaluator.id]
-                    evaluator_averages[evaluator.id] += (
-                        dto_result.score - evaluator_averages[evaluator.id]
-                    ) / count
-
-                    evaluation_run_results.evaluation_run_results.append(
-                        EvaluationRunResultDto(
-                            evaluator_name=evaluator.name,
-                            result=dto_result,
-                        )
-                    )
-                    evaluation_item_results.append(
-                        EvalItemResult(
-                            evaluator_id=evaluator.id,
-                            result=evaluation_result,
-                        )
-                    )
-
-                evaluation_run_results.compute_average_score()
-
-                await event_bus.publish(
-                    EvaluationEvents.UPDATE_EVAL_RUN,
-                    EvalRunUpdatedEvent(
-                        execution_id=self.execution_id,
-                        eval_item=eval_item,
-                        eval_results=evaluation_item_results,
-                        success=not agent_execution_output.result.error,
-                        agent_output=agent_execution_output.result.output,
-                        agent_execution_time=agent_execution_output.execution_time,
-                        spans=agent_execution_output.spans,
-                        logs=agent_execution_output.logs,
-                    ),
-                    wait_for_completion=False,
-                )
-            except Exception as e:
-                exception_details = EvalItemExceptionDetails(exception=e)
+        # Computing evaluator averages
+        evaluator_averages: Dict[str, float] = defaultdict(float)
+        evaluator_count: Dict[str, int] = defaultdict(int)
 
-                for evaluator in evaluators:
-                    evaluator_counts[evaluator.id] += 1
-                    count = evaluator_counts[evaluator.id]
-                    evaluator_averages[evaluator.id] += (
-                        0.0 - evaluator_averages[evaluator.id]
-                    ) / count
+        for eval_run_result in results.evaluation_set_results:
+            for result_dto in eval_run_result.evaluation_run_results:
+                evaluator_averages[result_dto.evaluator_id] += result_dto.result.score
+                evaluator_count[result_dto.evaluator_id] += 1
 
-                eval_run_updated_event = EvalRunUpdatedEvent(
-                    execution_id=self.execution_id,
-                    eval_item=eval_item,
-                    eval_results=[],
-                    success=False,
-                    agent_output={},
-                    agent_execution_time=0.0,
-                    exception_details=exception_details,
-                    spans=[],
-                    logs=[],
-                )
-                if isinstance(e, EvaluationRuntimeException):
-                    eval_run_updated_event.spans = e.spans
-                    eval_run_updated_event.logs = e.logs
-                    eval_run_updated_event.exception_details.exception = (  # type: ignore
-                        e.root_exception
-                    )
-                    eval_run_updated_event.exception_details.runtime_exception = True  # type: ignore
-
-                await event_bus.publish(
-                    EvaluationEvents.UPDATE_EVAL_RUN,
-                    eval_run_updated_event,
-                    wait_for_completion=False,
-                )
-
-        results.compute_average_score()
+        for eval_id in evaluator_averages:
+            evaluator_averages[eval_id] = (
+                evaluator_averages[eval_id] / evaluator_count[eval_id]
+            )
 
         await event_bus.publish(
             EvaluationEvents.UPDATE_EVAL_SET_RUN,
@@ -285,6 +210,182 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
         )
         return self.context.result
 
+    async def _execute_sequential(
+        self,
+        evaluation_set: EvaluationSet,
+        evaluators: List[BaseEvaluator[Any]],
+        event_bus: EventBus,
+    ) -> List[EvaluationRunResult]:
+        all_eval_run_result: list[EvaluationRunResult] = []
+
+        for eval_item in evaluation_set.evaluations:
+            all_eval_run_result.append(
+                await self._execute_eval(eval_item, evaluators, event_bus)
+            )
+
+        return all_eval_run_result
+
+    async def _execute_parallel(
+        self,
+        evaluation_set: EvaluationSet,
+        evaluators: List[BaseEvaluator[Any]],
+        event_bus: EventBus,
+        workers: int,
+    ) -> List[EvaluationRunResult]:
+        # Create a queue with max concurrency
+        queue: asyncio.Queue[tuple[int, EvaluationItem]] = asyncio.Queue(
+            maxsize=workers
+        )
+
+        # Dictionary to store results with their original indices
+        results_dict: Dict[int, EvaluationRunResult] = {}
+
+        # Producer task to fill the queue
+        async def producer() -> None:
+            for index, eval_item in enumerate(evaluation_set.evaluations):
+                await queue.put((index, eval_item))
+            # Signal completion by putting None markers
+            for _ in range(workers):
+                await queue.put(None)  # type: ignore
+
+        # Worker function to process items from the queue
+        async def worker(worker_id: int) -> None:
+            while True:
+                item = await queue.get()
+
+                # Check for termination signal
+                if item is None:
+                    queue.task_done()
+                    break
+
+                index, eval_item = item
+
+                try:
+                    # Execute the evaluation
+                    result = await self._execute_eval(eval_item, evaluators, event_bus)
+
+                    # Store result with its index to maintain order
+                    results_dict[index] = result
+                finally:
+                    # Mark the task as done
+                    queue.task_done()
+
+        # Start producer
+        producer_task = asyncio.create_task(producer())
+
+        # Create worker tasks based on workers
+        worker_tasks = [asyncio.create_task(worker(i)) for i in range(workers)]
+
+        # Wait for producer and all workers to complete
+        await producer_task
+        await asyncio.gather(*worker_tasks)
+
+        # Return results in the original order
+        return [results_dict[i] for i in range(len(evaluation_set.evaluations))]
+
+    async def _execute_eval(
+        self,
+        eval_item: EvaluationItem,
+        evaluators: List[BaseEvaluator[Any]],
+        event_bus: EventBus,
+    ) -> EvaluationRunResult:
+        set_evaluation_item(eval_item)
+
+        await event_bus.publish(
+            EvaluationEvents.CREATE_EVAL_RUN,
+            EvalRunCreatedEvent(
+                execution_id=self.execution_id,
+                eval_item=eval_item,
+            ),
+        )
+
+        evaluation_run_results = EvaluationRunResult(
+            evaluation_name=eval_item.name, evaluation_run_results=[]
+        )
+
+        try:
+            agent_execution_output = await self.execute_runtime(eval_item)
+            evaluation_item_results: list[EvalItemResult] = []
+
+            for evaluator in evaluators:
+                evaluation_result = await self.run_evaluator(
+                    evaluator=evaluator,
+                    execution_output=agent_execution_output,
+                    eval_item=eval_item,
+                )
+
+                dto_result = EvaluationResultDto.from_evaluation_result(
+                    evaluation_result
+                )
+
+                evaluation_run_results.evaluation_run_results.append(
+                    EvaluationRunResultDto(
+                        evaluator_name=evaluator.name,
+                        result=dto_result,
+                        evaluator_id=evaluator.id,
+                    )
+                )
+                evaluation_item_results.append(
+                    EvalItemResult(
+                        evaluator_id=evaluator.id,
+                        result=evaluation_result,
+                    )
+                )
+
+            await event_bus.publish(
+                EvaluationEvents.UPDATE_EVAL_RUN,
+                EvalRunUpdatedEvent(
+                    execution_id=self.execution_id,
+                    eval_item=eval_item,
+                    eval_results=evaluation_item_results,
+                    success=not agent_execution_output.result.error,
+                    agent_output=agent_execution_output.result.output,
+                    agent_execution_time=agent_execution_output.execution_time,
+                    spans=agent_execution_output.spans,
+                    logs=agent_execution_output.logs,
+                ),
+                wait_for_completion=False,
+            )
+
+        except Exception as e:
+            exception_details = EvalItemExceptionDetails(exception=e)
+
+            for evaluator in evaluators:
+                evaluation_run_results.evaluation_run_results.append(
+                    EvaluationRunResultDto(
+                        evaluator_name=evaluator.name,
+                        evaluator_id=evaluator.id,
+                        result=EvaluationResultDto(score=0),
+                    )
+                )
+
+            eval_run_updated_event = EvalRunUpdatedEvent(
+                execution_id=self.execution_id,
+                eval_item=eval_item,
+                eval_results=[],
+                success=False,
+                agent_output={},
+                agent_execution_time=0.0,
+                exception_details=exception_details,
+                spans=[],
+                logs=[],
+            )
+            if isinstance(e, EvaluationRuntimeException):
+                eval_run_updated_event.spans = e.spans
+                eval_run_updated_event.logs = e.logs
+                eval_run_updated_event.exception_details.exception = (  # type: ignore
+                    e.root_exception
+                )
+                eval_run_updated_event.exception_details.runtime_exception = True  # type: ignore
+
+            await event_bus.publish(
+                EvaluationEvents.UPDATE_EVAL_RUN,
+                eval_run_updated_event,
+                wait_for_completion=False,
+            )
+
+        return evaluation_run_results
+
     def _get_and_clear_execution_data(
         self, execution_id: str
     ) -> tuple[List[ReadableSpan], list[logging.LogRecord]]:

uipath/_cli/_runtime/_runtime.py

@@ -45,7 +45,9 @@ class UiPathRuntime(UiPathBaseRuntime):
         try:
             script_result = await self.executor(self.context.input_json)
 
-            if self.context.job_id is None:
+            if self.context.job_id is None and not getattr(
+                self.context, "is_eval_run", False
+            ):
                 logger.info(script_result)
 
             self.context.result = UiPathRuntimeResult(

uipath/_cli/_utils/_common.py

@@ -12,19 +12,19 @@ from ..spinner import Spinner
 def environment_options(function):
     function = click.option(
         "--alpha",
-        "domain",
+        "environment",
         flag_value="alpha",
         help="Use alpha environment",
     )(function)
     function = click.option(
         "--staging",
-        "domain",
+        "environment",
         flag_value="staging",
         help="Use staging environment",
     )(function)
     function = click.option(
         "--cloud",
-        "domain",
+        "environment",
         flag_value="cloud",
         default=True,
         help="Use production environment",

uipath/_cli/cli_auth.py

@@ -48,7 +48,7 @@ console = ConsoleLogger()
 )
 @track
 def auth(
-    domain,
+    environment: str,
     force: bool = False,
     client_id: Optional[str] = None,
     client_secret: Optional[str] = None,
@@ -70,7 +70,7 @@ def auth(
     - Set UIPATH_DISABLE_SSL_VERIFY to disable SSL verification (not recommended)
     """
     auth_service = AuthService(
-        domain,
+        environment=environment,
         force=force,
         client_id=client_id,
         client_secret=client_secret,

uipath/_cli/cli_eval.py

@@ -69,8 +69,8 @@ def setup_reporting_prereq(no_report: bool) -> bool:
 @click.option(
     "--workers",
     type=int,
-    default=8,
-    help="Number of parallel workers for running evaluations (default: 8)",
+    default=1,
+    help="Number of parallel workers for running evaluations (default: 1)",
 )
 @click.option(
     "--output-file",

uipath/_services/__init__.py

@@ -7,6 +7,7 @@ from .connections_service import ConnectionsService
 from .context_grounding_service import ContextGroundingService
 from .documents_service import DocumentsService
 from .entities_service import EntitiesService
+from .external_application_service import ExternalApplicationService
 from .folder_service import FolderService
 from .jobs_service import JobsService
 from .llm_gateway_service import UiPathLlmChatService, UiPathOpenAIService
@@ -29,4 +30,5 @@ __all__ = [
     "UiPathLlmChatService",
     "FolderService",
     "EntitiesService",
+    "ExternalApplicationService",
 ]

uipath/{_cli/_auth/_client_credentials.py → _services/external_application_service.py}

@@ -1,22 +1,26 @@
-from typing import Optional, cast
+from os import environ as env
+from typing import Optional
 from urllib.parse import urlparse
 
 import httpx
+from httpx import HTTPStatusError, Request
 
-from ..._utils._ssl_context import get_httpx_client_kwargs
-from .._utils._console import ConsoleLogger
-from ._models import TokenData
-from ._utils import parse_access_token, update_env_file
+from .._utils._ssl_context import get_httpx_client_kwargs
+from .._utils.constants import ENV_BASE_URL
+from ..models.auth import TokenData
+from ..models.exceptions import EnrichedException
 
-console = ConsoleLogger()
 
-
-class ClientCredentialsService:
+class ExternalApplicationService:
     """Service for client credentials authentication flow."""
 
-    def __init__(self, base_url: str):
-        self._base_url = base_url
-        self._domain = self._extract_domain_from_base_url(base_url)
+    def __init__(self, base_url: Optional[str]):
+        if not (resolved_base_url := (base_url or env.get(ENV_BASE_URL))):
+            raise ValueError(
+                "Base URL must be set either via constructor or the BASE_URL environment variable."
+            )
+        self._base_url = resolved_base_url
+        self._domain = self._extract_environment_from_base_url(self._base_url)
 
     def get_token_url(self) -> str:
         """Get the token URL for the specified domain."""
@@ -40,7 +44,7 @@ class ClientCredentialsService:
         """
         return hostname == domain or hostname.endswith(f".{domain}")
 
-    def _extract_domain_from_base_url(self, base_url: str) -> str:
+    def _extract_environment_from_base_url(self, base_url: str) -> str:
         """Extract domain from base URL.
 
         Args:
@@ -70,9 +74,9 @@ class ClientCredentialsService:
             # Default to cloud if parsing fails
             return "cloud"
 
-    def authenticate(
+    def get_token_data(
         self, client_id: str, client_secret: str, scope: Optional[str] = "OR.Execution"
-    ) -> None:
+    ) -> TokenData:
         """Authenticate using client credentials flow.
 
         Args:
@@ -81,7 +85,7 @@ class ClientCredentialsService:
             scope: The scope for the token (default: OR.Execution)
 
         Returns:
-            Token data if successful, None otherwise
+            Token data if successful
         """
         token_url = self.get_token_url()
 
@@ -97,49 +101,40 @@ class ClientCredentialsService:
                 response = client.post(token_url, data=data)
                 match response.status_code:
                     case 200:
-                        token_data = response.json()
-                        token_data = cast(
-                            TokenData,
-                            {
-                                "access_token": token_data["access_token"],
-                                "token_type": token_data.get("token_type", "Bearer"),
-                                "expires_in": token_data.get("expires_in", 3600),
-                                "scope": token_data.get("scope", scope),
-                                "refresh_token": "",
-                                "id_token": "",
-                            },
-                        )
-                        self._setup_environment(token_data)
+                        return TokenData.model_validate(response.json())
                     case 400:
-                        console.error(
-                            "Invalid client credentials or request parameters."
+                        raise EnrichedException(
+                            HTTPStatusError(
+                                message="Invalid client credentials or request parameters.",
+                                request=Request(
+                                    data=data, url=token_url, method="post"
+                                ),
+                                response=response,
+                            )
                         )
                     case 401:
-                        console.error("Unauthorized: Invalid client credentials.")
+                        raise EnrichedException(
+                            HTTPStatusError(
+                                message="Unauthorized: Invalid client credentials.",
+                                request=Request(
+                                    data=data, url=token_url, method="post"
+                                ),
+                                response=response,
+                            )
+                        )
                     case _:
-                        console.error(
-                            f"Authentication failed: {response.status_code} - {response.text}"
+                        raise EnrichedException(
+                            HTTPStatusError(
+                                message=f"Authentication failed with unexpected status: {response.status_code}",
+                                request=Request(
+                                    data=data, url=token_url, method="post"
+                                ),
+                                response=response,
+                            )
                         )
-
+        except EnrichedException:
+            raise
         except httpx.RequestError as e:
-            console.error(f"Network error during authentication: {e}")
+            raise Exception(f"Network error during authentication: {e}") from e
         except Exception as e:
-            console.error(f"Unexpected error during authentication: {e}")
-
-    def _setup_environment(self, token_data: TokenData):
-        """Setup environment variables for client credentials authentication.
-
-        Args:
-            token_data: The token data from authentication
-            base_url: The base URL for the UiPath instance
-        """
-        parsed_access_token = parse_access_token(token_data["access_token"])
-
-        env_vars = {
-            "UIPATH_ACCESS_TOKEN": token_data["access_token"],
-            "UIPATH_URL": self._base_url,
-            "UIPATH_ORGANIZATION_ID": parsed_access_token.get("prt_id", ""),
-            "UIPATH_TENANT_ID": "",
-        }
-
-        update_env_file(env_vars)
+            raise Exception(f"Unexpected error during authentication: {e}") from e

uipath/_uipath.py

@@ -1,4 +1,3 @@
-from os import environ as env
 from typing import Optional
 
 from pydantic import ValidationError
@@ -23,11 +22,7 @@ from ._services import (
     UiPathOpenAIService,
 )
 from ._utils import setup_logging
-from ._utils.constants import (
-    ENV_BASE_URL,
-    ENV_UIPATH_ACCESS_TOKEN,
-    ENV_UNATTENDED_USER_ACCESS_TOKEN,
-)
+from ._utils._auth import resolve_config
 from .models.errors import BaseUrlMissingError, SecretMissingError
 
 
@@ -37,19 +32,18 @@ class UiPath:
         *,
         base_url: Optional[str] = None,
         secret: Optional[str] = None,
+        client_id: Optional[str] = None,
+        client_secret: Optional[str] = None,
+        scope: Optional[str] = None,
         debug: bool = False,
     ) -> None:
-        base_url_value = base_url or env.get(ENV_BASE_URL)
-        secret_value = (
-            secret
-            or env.get(ENV_UNATTENDED_USER_ACCESS_TOKEN)
-            or env.get(ENV_UIPATH_ACCESS_TOKEN)
-        )
-
         try:
+            base_url, secret = resolve_config(
+                base_url, secret, client_id, client_secret, scope
+            )
             self._config = Config(
-                base_url=base_url_value,  # type: ignore
-                secret=secret_value,  # type: ignore
+                base_url=base_url,
+                secret=secret,
             )
         except ValidationError as e:
             for error in e.errors():