uipath 2.1.60__py3-none-any.whl → 2.1.61__py3-none-any.whl

uipath/_cli/_runtime/_contracts.py

@@ -753,17 +753,24 @@ class UiPathRuntimeFactory(Generic[T, C]):
                     span_processor.force_flush()
 
     async def execute_in_root_span(
-        self, context: C, root_span: str = "root"
+        self,
+        context: C,
+        root_span: str = "root",
+        attributes: Optional[dict[str, str]] = None,
     ) -> Optional[UiPathRuntimeResult]:
         """Execute runtime with context."""
         async with self.from_context(context) as runtime:
             try:
                 tracer: Tracer = trace.get_tracer("uipath-runtime")
+                span_attributes = {}
+                if context.execution_id:
+                    span_attributes["execution.id"] = context.execution_id
+                if attributes:
+                    span_attributes.update(attributes)
+
                 with tracer.start_as_current_span(
                     root_span,
-                    attributes={"execution.id": context.execution_id}
-                    if context.execution_id
-                    else {},
+                    attributes=span_attributes,
                 ):
                     return await runtime.execute()
             finally:

uipath/_cli/_utils/_folders.py

@@ -1,35 +1,41 @@
-from typing import Optional, Tuple
+from typing import Any, Optional, Tuple
 
-import httpx
-
-from ..._utils._ssl_context import get_httpx_client_kwargs
 from ._console import ConsoleLogger
 
 console = ConsoleLogger()
 
 
-def get_personal_workspace_info(
-    base_url: str, token: str
-) -> Tuple[Optional[str], Optional[str]]:
-    user_url = f"{base_url}/orchestrator_/odata/Users/UiPath.Server.Configuration.OData.GetCurrentUserExtended?$expand=PersonalWorkspace"
+async def get_personal_workspace_info_async() -> Tuple[Optional[str], Optional[str]]:
+    response = await _get_personal_workspace_info_internal_async()
+    feed_id = response.get("PersonalWorskpaceFeedId")
+    personal_workspace = response.get("PersonalWorkspace")
+
+    if not personal_workspace or not feed_id or "Id" not in personal_workspace:
+        return None, None
+
+    folder_id = personal_workspace.get("Id")
+    return feed_id, folder_id
+
+
+async def get_personal_workspace_key_async() -> Optional[str]:
+    response = await _get_personal_workspace_info_internal_async()
+    personal_workspace = response.get("PersonalWorkspace")
+    if not personal_workspace or "Key" not in personal_workspace:
+        return None
+    return personal_workspace["Key"]
+
 
-    with httpx.Client(**get_httpx_client_kwargs()) as client:
-        user_response = client.get(
-            user_url, headers={"Authorization": f"Bearer {token}"}
-        )
+async def _get_personal_workspace_info_internal_async() -> dict[str, Any]:
+    from ... import UiPath
 
-        if user_response.status_code != 200:
-            console.error(
-                "Error: Failed to fetch user info. Please try reauthenticating."
-            )
-            return None, None
+    uipath = UiPath()
 
-        user_data = user_response.json()
-        feed_id = user_data.get("PersonalWorskpaceFeedId")
-        personal_workspace = user_data.get("PersonalWorkspace")
+    response = await uipath.api_client.request_async(
+        method="GET",
+        url="/orchestrator_/odata/Users/UiPath.Server.Configuration.OData.GetCurrentUserExtended?$expand=PersonalWorkspace",
+    )
 
-        if not personal_workspace or not feed_id or "Id" not in personal_workspace:
-            return None, None
+    if response.status_code != 200:
+        console.error("Error: Failed to fetch user info. Please try reauthenticating.")
 
-        folder_id = personal_workspace.get("Id")
-        return feed_id, folder_id
+    return response.json()

uipath/_cli/cli_eval.py

@@ -2,17 +2,24 @@
 import ast
 import asyncio
 import os
+import uuid
 from typing import List, Optional
 
 import click
 
-from uipath._cli._evals._runtime import UiPathEvalContext, UiPathEvalRuntime
+from uipath._cli._evals._progress_reporter import StudioWebProgressReporter
+from uipath._cli._evals._runtime import (
+    UiPathEvalContext,
+    UiPathEvalRuntime,
+)
 from uipath._cli._runtime._contracts import (
     UiPathRuntimeContext,
     UiPathRuntimeFactory,
 )
 from uipath._cli._runtime._runtime import UiPathScriptRuntime
+from uipath._cli._utils._folders import get_personal_workspace_key_async
 from uipath._cli.middlewares import Middlewares
+from uipath._events._event_bus import EventBus
 from uipath.eval._helpers import auto_discover_entrypoint
 from uipath.tracing import LlmOpsHttpExporter
 
@@ -72,6 +79,11 @@ def eval(
         workers: Number of parallel workers for running evaluations
         no_report: Do not report the evaluation results
     """
+    if not no_report and not os.getenv("UIPATH_FOLDER_KEY"):
+        os.environ["UIPATH_FOLDER_KEY"] = asyncio.run(
+            get_personal_workspace_key_async()
+        )
+
     result = Middlewares.next(
         "eval",
         entrypoint,
@@ -86,14 +98,23 @@ def eval(
         console.error(result.error_message)
 
     if result.should_continue:
+        event_bus = EventBus()
+
+        if not no_report:
+            progress_reporter = StudioWebProgressReporter()
+            asyncio.run(progress_reporter.subscribe_to_eval_runtime_events(event_bus))
 
         def generate_runtime_context(**context_kwargs) -> UiPathRuntimeContext:
             runtime_context = UiPathRuntimeContext.with_defaults(**context_kwargs)
             runtime_context.entrypoint = runtime_entrypoint
             return runtime_context
 
+        runtime_entrypoint = entrypoint or auto_discover_entrypoint()
+
         eval_context = UiPathEvalContext.with_defaults(
-            execution_output_file=output_file
+            execution_output_file=output_file,
+            entrypoint=runtime_entrypoint,
+            execution_id=str(uuid.uuid4()),
         )
 
         eval_context.no_report = no_report
@@ -101,22 +122,23 @@ def eval(
         eval_context.eval_set = eval_set or EvalHelpers.auto_discover_eval_set()
         eval_context.eval_ids = eval_ids
 
-        runtime_entrypoint = entrypoint or auto_discover_entrypoint()
-
         try:
             runtime_factory = UiPathRuntimeFactory(
                 UiPathScriptRuntime,
                 UiPathRuntimeContext,
                 context_generator=generate_runtime_context,
             )
-            if not no_report or eval_context.job_id:
+            if eval_context.job_id:
                 runtime_factory.add_span_exporter(LlmOpsHttpExporter())
 
             async def execute():
                 async with UiPathEvalRuntime.from_eval_context(
-                    factory=runtime_factory, context=eval_context
+                    factory=runtime_factory,
+                    context=eval_context,
+                    event_bus=event_bus,
                 ) as eval_runtime:
                     await eval_runtime.execute()
+                    await event_bus.wait_for_all(timeout=10)
 
             asyncio.run(execute())
         except Exception as e:

uipath/_cli/cli_invoke.py

@@ -1,4 +1,5 @@
 # type: ignore
+import asyncio
 import logging
 import os
 from typing import Optional
@@ -16,7 +17,7 @@ except ImportError:
 from .._utils._ssl_context import get_httpx_client_kwargs
 from ..telemetry import track
 from ._utils._common import get_env_vars
-from ._utils._folders import get_personal_workspace_info
+from ._utils._folders import get_personal_workspace_info_async
 from ._utils._processes import get_release_info
 from .middlewares import Middlewares
 
@@ -65,7 +66,9 @@ def invoke(
         [base_url, token] = get_env_vars()
 
         url = f"{base_url}/orchestrator_/odata/Jobs/UiPath.Server.Configuration.OData.StartJobs"
-        _, personal_workspace_folder_id = get_personal_workspace_info(base_url, token)
+        _, personal_workspace_folder_id = asyncio.run(
+            get_personal_workspace_info_async()
+        )
         project_name, project_version = _read_project_details()
         if not personal_workspace_folder_id:
             console.error(

uipath/_cli/cli_publish.py

@@ -1,4 +1,5 @@
 # type: ignore
+import asyncio
 import json
 import os
 
@@ -9,7 +10,7 @@ from .._utils._ssl_context import get_httpx_client_kwargs
 from ..telemetry import track
 from ._utils._common import get_env_vars
 from ._utils._console import ConsoleLogger
-from ._utils._folders import get_personal_workspace_info
+from ._utils._folders import get_personal_workspace_info_async
 from ._utils._processes import get_release_info
 
 console = ConsoleLogger()
@@ -106,8 +107,8 @@ def publish(feed):
 
         if feed and feed != "tenant":
             # Check user personal workspace
-            personal_workspace_feed_id, personal_workspace_folder_id = (
-                get_personal_workspace_info(base_url, token)
+            personal_workspace_feed_id, personal_workspace_folder_id = asyncio.run(
+                get_personal_workspace_info_async()
             )
             if feed == "personal" or feed == personal_workspace_feed_id:
                 is_personal_workspace = True

uipath/_events/_event_bus.py

@@ -0,0 +1,157 @@
+"""Event bus implementation for evaluation events."""
+
+import asyncio
+import logging
+from typing import Any, Callable, Dict, List, Optional, Set, TypeVar
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+
+class EventBus:
+    """Event bus for publishing and subscribing to events."""
+
+    def __init__(self) -> None:
+        """Initialize a new EventBus instance."""
+        self._subscribers: Dict[str, List[Callable[[Any], Any]]] = {}
+        self._running_tasks: Set[asyncio.Task[Any]] = set()
+
+    def subscribe(self, topic: str, handler: Callable[[Any], Any]) -> None:
+        """Subscribe a handler method/function to a topic.
+
+        Args:
+            topic: The topic name to subscribe to.
+            handler: The async handler method/function that will handle events for this topic.
+        """
+        if topic not in self._subscribers:
+            self._subscribers[topic] = []
+        self._subscribers[topic].append(handler)
+        logger.debug(f"Handler registered for topic: {topic}")
+
+    def unsubscribe(self, topic: str, handler: Callable[[Any], Any]) -> None:
+        """Unsubscribe a handler from a topic.
+
+        Args:
+            topic: The topic name to unsubscribe from.
+            handler: The handler to remove.
+        """
+        if topic in self._subscribers:
+            try:
+                self._subscribers[topic].remove(handler)
+                if not self._subscribers[topic]:
+                    del self._subscribers[topic]
+                logger.debug(f"Handler unregistered from topic: {topic}")
+            except ValueError:
+                logger.warning(f"Handler not found for topic: {topic}")
+
+    def _cleanup_completed_tasks(self) -> None:
+        completed_tasks = {task for task in self._running_tasks if task.done()}
+        self._running_tasks -= completed_tasks
+
+    async def publish(
+        self, topic: str, payload: T, wait_for_completion: bool = True
+    ) -> None:
+        """Publish an event to all handlers of a topic.
+
+        Args:
+            topic: The topic name to publish to.
+            payload: The event payload to publish.
+            wait_for_completion: Whether to wait for the event to be processed.
+        """
+        if topic not in self._subscribers:
+            logger.debug(f"No handlers for topic: {topic}")
+            return
+
+        self._cleanup_completed_tasks()
+
+        tasks = []
+        for subscriber in self._subscribers[topic]:
+            try:
+                task = asyncio.create_task(subscriber(payload))
+                tasks.append(task)
+                self._running_tasks.add(task)
+            except Exception as e:
+                logger.error(f"Error creating task for subscriber {subscriber}: {e}")
+
+        if tasks and wait_for_completion:
+            try:
+                await asyncio.gather(*tasks, return_exceptions=True)
+            except Exception as e:
+                logger.error(f"Error during event processing for topic {topic}: {e}")
+            finally:
+                # Clean up the tasks we just waited for
+                for task in tasks:
+                    self._running_tasks.discard(task)
+
+    def get_running_tasks_count(self) -> int:
+        """Get the number of currently running subscriber tasks.
+
+        Returns:
+            Number of running tasks.
+        """
+        self._cleanup_completed_tasks()
+        return len(self._running_tasks)
+
+    async def wait_for_all(self, timeout: Optional[float] = None) -> None:
+        """Wait for all currently running subscriber tasks to complete.
+
+        Args:
+            timeout: Maximum time to wait in seconds. If None, waits indefinitely.
+        """
+        self._cleanup_completed_tasks()
+
+        if not self._running_tasks:
+            logger.debug("No running tasks to wait for")
+            return
+
+        logger.debug(
+            f"Waiting for {len(self._running_tasks)} EventBus tasks to complete..."
+        )
+
+        try:
+            tasks_to_wait = list(self._running_tasks)
+
+            if timeout:
+                await asyncio.wait_for(
+                    asyncio.gather(*tasks_to_wait, return_exceptions=True),
+                    timeout=timeout,
+                )
+            else:
+                await asyncio.gather(*tasks_to_wait, return_exceptions=True)
+
+            logger.debug("All EventBus tasks completed")
+
+        except asyncio.TimeoutError:
+            logger.warning(f"Timeout waiting for EventBus tasks after {timeout}s")
+            for task in tasks_to_wait:
+                if not task.done():
+                    task.cancel()
+        except Exception as e:
+            logger.error(f"Error waiting for EventBus tasks: {e}")
+        finally:
+            self._cleanup_completed_tasks()
+
+    def get_subscribers_count(self, topic: str) -> int:
+        """Get the number of subscribers for a topic.
+
+        Args:
+            topic: The topic name.
+
+        Returns:
+            Number of handlers for the topic.
+        """
+        return len(self._subscribers.get(topic, []))
+
+    def clear_subscribers(self, topic: Optional[str] = None) -> None:
+        """Clear subscribers for a topic or all topics.
+
+        Args:
+            topic: The topic to clear. If None, clears all topics.
+        """
+        if topic is None:
+            self._subscribers.clear()
+            logger.debug("All handlers cleared")
+        elif topic in self._subscribers:
+            del self._subscribers[topic]
+            logger.debug(f"Handlers cleared for topic: {topic}")

uipath/_events/_events.py

@@ -0,0 +1,53 @@
+import enum
+from typing import Any, List, Union
+
+from opentelemetry.sdk.trace import ReadableSpan
+from pydantic import BaseModel, ConfigDict
+
+from uipath._cli._evals._models._evaluation_set import EvaluationItem
+from uipath.eval.models import EvalItemResult
+
+
+class EvaluationEvents(str, enum.Enum):
+    CREATE_EVAL_SET_RUN = "create_eval_set_run"
+    CREATE_EVAL_RUN = "create_eval_run"
+    UPDATE_EVAL_SET_RUN = "update_eval_set_run"
+    UPDATE_EVAL_RUN = "update_eval_run"
+
+
+class EvalSetRunCreatedEvent(BaseModel):
+    execution_id: str
+    entrypoint: str
+    eval_set_id: str
+    no_of_evals: int
+    evaluators: List[Any]
+
+
+class EvalRunCreatedEvent(BaseModel):
+    execution_id: str
+    eval_item: EvaluationItem
+
+
+class EvalRunUpdatedEvent(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    execution_id: str
+    eval_item: EvaluationItem
+    eval_results: List[EvalItemResult]
+    success: bool
+    agent_output: Any
+    agent_execution_time: float
+    spans: List[ReadableSpan]
+
+
+class EvalSetRunUpdatedEvent(BaseModel):
+    execution_id: str
+    evaluator_scores: dict[str, float]
+
+
+ProgressEvent = Union[
+    EvalSetRunCreatedEvent,
+    EvalRunCreatedEvent,
+    EvalRunUpdatedEvent,
+    EvalSetRunUpdatedEvent,
+]

uipath/eval/models/models.py

@@ -70,7 +70,7 @@ EvaluationResult = Annotated[
 class EvalItemResult(BaseModel):
     """Result of a single evaluation item."""
 
-    evaluator_name: str
+    evaluator_id: str
     result: EvaluationResult
 
 

uipath/tracing/_otel_exporters.py

@@ -1,91 +1,95 @@
-import json
-import logging
-import os
-import time
-from typing import Any, Dict, Sequence
-
-import httpx
-from opentelemetry.sdk.trace import ReadableSpan
-from opentelemetry.sdk.trace.export import (
-    SpanExporter,
-    SpanExportResult,
-)
-
-from uipath._utils._ssl_context import get_httpx_client_kwargs
-
-from ._utils import _SpanUtils
-
-logger = logging.getLogger(__name__)
-
-
-class LlmOpsHttpExporter(SpanExporter):
-    """An OpenTelemetry span exporter that sends spans to UiPath LLM Ops."""
-
-    def __init__(self, **client_kwargs):
-        """Initialize the exporter with the base URL and authentication token."""
-        super().__init__(**client_kwargs)
-        self.base_url = self._get_base_url()
-        self.auth_token = os.environ.get("UIPATH_ACCESS_TOKEN")
-        self.headers = {
-            "Content-Type": "application/json",
-            "Authorization": f"Bearer {self.auth_token}",
-        }
-
-        client_kwargs = get_httpx_client_kwargs()
-
-        self.http_client = httpx.Client(**client_kwargs, headers=self.headers)
-
-    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
-        """Export spans to UiPath LLM Ops."""
-        logger.debug(
-            f"Exporting {len(spans)} spans to {self.base_url}/llmopstenant_/api/Traces/spans"
-        )
-
-        span_list = [
-            _SpanUtils.otel_span_to_uipath_span(span).to_dict() for span in spans
-        ]
-        url = self._build_url(span_list)
-
-        logger.debug("Payload: %s", json.dumps(span_list))
-
-        return self._send_with_retries(url, span_list)
-
-    def force_flush(self, timeout_millis: int = 30000) -> bool:
-        """Force flush the exporter."""
-        return True
-
-    def _build_url(self, span_list: list[Dict[str, Any]]) -> str:
-        """Construct the URL for the API request."""
-        trace_id = str(span_list[0]["TraceId"])
-        return f"{self.base_url}/llmopstenant_/api/Traces/spans?traceId={trace_id}&source=Robots"
-
-    def _send_with_retries(
-        self, url: str, payload: list[Dict[str, Any]], max_retries: int = 4
-    ) -> SpanExportResult:
-        """Send the HTTP request with retry logic."""
-        for attempt in range(max_retries):
-            try:
-                response = self.http_client.post(url, json=payload)
-                if response.status_code == 200:
-                    return SpanExportResult.SUCCESS
-                else:
-                    logger.warning(
-                        f"Attempt {attempt + 1} failed with status code {response.status_code}: {response.text}"
-                    )
-            except Exception as e:
-                logger.error(f"Attempt {attempt + 1} failed with exception: {e}")
-
-            if attempt < max_retries - 1:
-                time.sleep(1.5**attempt)  # Exponential backoff
-
-        return SpanExportResult.FAILURE
-
-    def _get_base_url(self) -> str:
-        uipath_url = (
-            os.environ.get("UIPATH_URL")
-            or "https://cloud.uipath.com/dummyOrg/dummyTennant/"
-        )
-
-        uipath_url = uipath_url.rstrip("/")
-
-        return uipath_url
+import json
+import logging
+import os
+import time
+from typing import Any, Dict, Optional, Sequence
+
+import httpx
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.sdk.trace.export import (
+    SpanExporter,
+    SpanExportResult,
+)
+
+from uipath._utils._ssl_context import get_httpx_client_kwargs
+
+from ._utils import _SpanUtils
+
+logger = logging.getLogger(__name__)
+
+
+class LlmOpsHttpExporter(SpanExporter):
+    """An OpenTelemetry span exporter that sends spans to UiPath LLM Ops."""
+
+    def __init__(self, trace_id: Optional[str] = None, **client_kwargs):
+        """Initialize the exporter with the base URL and authentication token."""
+        super().__init__(**client_kwargs)
+        self.base_url = self._get_base_url()
+        self.auth_token = os.environ.get("UIPATH_ACCESS_TOKEN")
+        self.headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.auth_token}",
+        }
+
+        client_kwargs = get_httpx_client_kwargs()
+
+        self.http_client = httpx.Client(**client_kwargs, headers=self.headers)
+        self.trace_id = trace_id
+
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        """Export spans to UiPath LLM Ops."""
+        logger.debug(
+            f"Exporting {len(spans)} spans to {self.base_url}/llmopstenant_/api/Traces/spans"
+        )
+
+        span_list = [
+            _SpanUtils.otel_span_to_uipath_span(
+                span, custom_trace_id=self.trace_id
+            ).to_dict()
+            for span in spans
+        ]
+        url = self._build_url(span_list)
+
+        logger.debug("Payload: %s", json.dumps(span_list))
+
+        return self._send_with_retries(url, span_list)
+
+    def force_flush(self, timeout_millis: int = 30000) -> bool:
+        """Force flush the exporter."""
+        return True
+
+    def _build_url(self, span_list: list[Dict[str, Any]]) -> str:
+        """Construct the URL for the API request."""
+        trace_id = str(span_list[0]["TraceId"])
+        return f"{self.base_url}/llmopstenant_/api/Traces/spans?traceId={trace_id}&source=Robots"
+
+    def _send_with_retries(
+        self, url: str, payload: list[Dict[str, Any]], max_retries: int = 4
+    ) -> SpanExportResult:
+        """Send the HTTP request with retry logic."""
+        for attempt in range(max_retries):
+            try:
+                response = self.http_client.post(url, json=payload)
+                if response.status_code == 200:
+                    return SpanExportResult.SUCCESS
+                else:
+                    logger.warning(
+                        f"Attempt {attempt + 1} failed with status code {response.status_code}: {response.text}"
+                    )
+            except Exception as e:
+                logger.error(f"Attempt {attempt + 1} failed with exception: {e}")
+
+            if attempt < max_retries - 1:
+                time.sleep(1.5**attempt)  # Exponential backoff
+
+        return SpanExportResult.FAILURE
+
+    def _get_base_url(self) -> str:
+        uipath_url = (
+            os.environ.get("UIPATH_URL")
+            or "https://cloud.uipath.com/dummyOrg/dummyTennant/"
+        )
+
+        uipath_url = uipath_url.rstrip("/")
+
+        return uipath_url