uipath 2.1.41__py3-none-any.whl → 2.1.43__py3-none-any.whl

uipath/_cli/_evals/_runtime.py

@@ -0,0 +1,172 @@
+import copy
+from collections import defaultdict
+from time import time
+from typing import Dict, Generic, List, Optional, Sequence, TypeVar
+
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
+
+from uipath.eval._helpers import auto_discover_entrypoint
+
+from .._runtime._contracts import (
+    UiPathBaseRuntime,
+    UiPathRuntimeContext,
+    UiPathRuntimeFactory,
+    UiPathRuntimeResult,
+    UiPathRuntimeStatus,
+)
+from .._utils._eval_set import EvalHelpers
+from ._models import EvaluationItem
+from ._models._agent_execution_output import UiPathEvalRunExecutionOutput
+
+T = TypeVar("T", bound=UiPathBaseRuntime)
+C = TypeVar("C", bound=UiPathRuntimeContext)
+
+
+class ExecutionSpanExporter(SpanExporter):
+    """Custom exporter that stores spans grouped by execution ids."""
+
+    def __init__(self):
+        # { execution_id -> list of spans }
+        self._spans: Dict[str, List[ReadableSpan]] = defaultdict(list)
+
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        for span in spans:
+            if span.attributes is not None:
+                exec_id = span.attributes.get("execution.id")
+                if exec_id is not None and isinstance(exec_id, str):
+                    self._spans[exec_id].append(span)
+
+        return SpanExportResult.SUCCESS
+
+    def get_spans(self, execution_id: str) -> List[ReadableSpan]:
+        """Retrieve spans for a given execution id."""
+        return self._spans.get(execution_id, [])
+
+    def clear(self, execution_id: Optional[str] = None) -> None:
+        """Clear stored spans for one or all executions."""
+        if execution_id:
+            self._spans.pop(execution_id, None)
+        else:
+            self._spans.clear()
+
+    def shutdown(self) -> None:
+        self.clear()
+
+
+class UiPathEvalContext(UiPathRuntimeContext, Generic[C]):
+    """Context used for evaluation runs."""
+
+    runtime_context: C
+    no_report: bool
+    workers: int
+    eval_set: Optional[str] = None
+    eval_ids: Optional[List[str]] = None
+
+    def __init__(
+        self,
+        runtime_context: C,
+        no_report: bool,
+        workers: int,
+        eval_set: Optional[str] = None,
+        eval_ids: Optional[List[str]] = None,
+        **kwargs,
+    ):
+        super().__init__(
+            runtime_context=runtime_context,  # type: ignore
+            no_report=no_report,
+            workers=workers,
+            eval_set=eval_set,
+            eval_ids=eval_ids,
+            **kwargs,
+        )
+        self._auto_discover()
+
+    def _auto_discover(self):
+        self.runtime_context.entrypoint = (
+            self.runtime_context.entrypoint or auto_discover_entrypoint()
+        )
+        self.eval_set = self.eval_set or EvalHelpers.auto_discover_eval_set()
+
+
+class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
+    """Specialized runtime for evaluation runs, with access to the factory."""
+
+    def __init__(
+        self, context: "UiPathEvalContext[C]", factory: "UiPathRuntimeFactory[T, C]"
+    ):
+        super().__init__(context)
+        self.context: "UiPathEvalContext[C]" = context
+        self.factory: UiPathRuntimeFactory[T, C] = factory
+        self.span_exporter: ExecutionSpanExporter = ExecutionSpanExporter()
+        self.factory.add_span_exporter(self.span_exporter)
+
+    @classmethod
+    def from__eval_context(
+        cls,
+        context: "UiPathEvalContext[C]",
+        factory: "UiPathRuntimeFactory[T, C]",
+    ) -> "UiPathEvalRuntime[T, C]":
+        return cls(context, factory)
+
+    async def execute(self) -> Optional[UiPathRuntimeResult]:
+        """Evaluation logic. Can spawn other runtimes through the factory."""
+        if self.context.eval_set is None:
+            raise ValueError("eval_set must be provided for evaluation runs")
+
+        evaluation_set = EvalHelpers.load_eval_set(
+            self.context.eval_set, self.context.eval_ids
+        )
+        execution_output_list: list[UiPathEvalRunExecutionOutput] = []
+        for eval_item in evaluation_set.evaluations:
+            execution_output_list.append(await self.execute_agent(eval_item))
+
+        self.context.result = UiPathRuntimeResult(
+            output={
+                "results": execution_output_list,
+            },
+            status=UiPathRuntimeStatus.SUCCESSFUL,
+            resume=None,
+        )
+
+        return self.context.runtime_context.result
+
+    def _prepare_new_runtime_context(self, eval_item: EvaluationItem) -> C:
+        runtime_context = copy.deepcopy(self.context.runtime_context)
+        runtime_context.execution_id = eval_item.id
+        runtime_context.input_json = eval_item.inputs
+        # here we can pass other values from eval_item: expectedAgentBehavior, simulationInstructions etc.
+        return runtime_context
+
+    # TODO: this would most likely need to be ported to a public AgentEvaluator class
+    async def execute_agent(
+        self, eval_item: EvaluationItem
+    ) -> "UiPathEvalRunExecutionOutput":
+        runtime_context = self._prepare_new_runtime_context(eval_item)
+        start_time = time()
+        result = await self.factory.execute_in_root_span(
+            runtime_context, root_span=eval_item.name
+        )
+        end_time = time()
+        if runtime_context.execution_id is None:
+            raise ValueError("execution_id must be set for eval runs")
+
+        spans = self.span_exporter.get_spans(runtime_context.execution_id)
+        self.span_exporter.clear(runtime_context.execution_id)
+
+        if result is None:
+            raise ValueError("Execution result cannot be None for eval runs")
+
+        return UiPathEvalRunExecutionOutput(
+            execution_time=end_time - start_time,
+            spans=spans,
+            result=result,
+        )
+
+    async def cleanup(self) -> None:
+        """Cleanup runtime resources."""
+        pass
+
+    async def validate(self) -> None:
+        """Cleanup runtime resources."""
+        pass

uipath/_cli/_runtime/_contracts.py

@@ -9,6 +9,7 @@ from abc import ABC, abstractmethod
 from enum import Enum
 from functools import cached_property
 from typing import Any, Callable, Dict, Generic, List, Optional, Type, TypeVar, Union
+from uuid import uuid4
 
 from opentelemetry import context as context_api
 from opentelemetry import trace
@@ -23,7 +24,7 @@ from opentelemetry.trace import Tracer
 from pydantic import BaseModel, Field
 
 from uipath.agent.conversation import UiPathConversationEvent, UiPathConversationMessage
-from uipath.tracing import TracingManager
+from uipath.tracing import LlmOpsHttpExporter, TracingManager
 
 from ._logging import LogsInterceptor
 
@@ -161,6 +162,128 @@ class UiPathTraceContext(BaseModel):
     reference_id: Optional[str] = None
 
 
+class UiPathRuntimeContextBuilder:
+    """Builder class for UiPathRuntimeContext following the builder pattern."""
+
+    def __init__(self):
+        self._kwargs = {}
+
+    def with_defaults(
+        self, config_path: Optional[str] = None, **kwargs
+    ) -> "UiPathRuntimeContextBuilder":
+        """Apply default configuration similar to UiPathRuntimeContext.with_defaults().
+
+        Args:
+            config_path: Path to the configuration file (defaults to UIPATH_CONFIG_PATH env var or "uipath.json")
+            **kwargs: Additional keyword arguments to pass to with_defaults
+
+        Returns:
+            Self for method chaining
+        """
+        from os import environ as env
+
+        resolved_config_path = config_path or env.get(
+            "UIPATH_CONFIG_PATH", "uipath.json"
+        )
+        self._kwargs["config_path"] = resolved_config_path
+
+        self._kwargs.update(
+            {
+                "job_id": env.get("UIPATH_JOB_KEY"),
+                "trace_id": env.get("UIPATH_TRACE_ID"),
+                "tracing_enabled": env.get("UIPATH_TRACING_ENABLED", True),
+                "logs_min_level": env.get("LOG_LEVEL", "INFO"),
+                **kwargs,  # Allow overriding defaults with provided kwargs
+            }
+        )
+
+        self._kwargs["trace_context"] = UiPathTraceContext(
+            trace_id=env.get("UIPATH_TRACE_ID"),
+            parent_span_id=env.get("UIPATH_PARENT_SPAN_ID"),
+            root_span_id=env.get("UIPATH_ROOT_SPAN_ID"),
+            enabled=env.get("UIPATH_TRACING_ENABLED", True),
+            job_id=env.get("UIPATH_JOB_KEY"),
+            org_id=env.get("UIPATH_ORGANIZATION_ID"),
+            tenant_id=env.get("UIPATH_TENANT_ID"),
+            process_key=env.get("UIPATH_PROCESS_UUID"),
+            folder_key=env.get("UIPATH_FOLDER_KEY"),
+            reference_id=env.get("UIPATH_JOB_KEY") or str(uuid4()),
+        )
+
+        return self
+
+    def with_entrypoint(self, entrypoint: str) -> "UiPathRuntimeContextBuilder":
+        """Set the entrypoint for the runtime context.
+
+        Args:
+            entrypoint: The entrypoint to execute
+
+        Returns:
+            Self for method chaining
+        """
+        self._kwargs["entrypoint"] = entrypoint
+        return self
+
+    def with_input(
+        self, input_data: Optional[str] = None, input_file: Optional[str] = None
+    ) -> "UiPathRuntimeContextBuilder":
+        """Set the input data for the runtime context.
+
+        Args:
+            input_data: The input data as a string
+            input_file: Path to the input file
+
+        Returns:
+            Self for method chaining
+        """
+        if input_data is not None:
+            self._kwargs["input"] = input_data
+        if input_file is not None:
+            self._kwargs["input_file"] = input_file
+        return self
+
+    def with_resume(self, enable: bool = True) -> "UiPathRuntimeContextBuilder":
+        """Enable or disable resume mode for the runtime context.
+
+        Args:
+            enable: Whether to enable resume mode (defaults to True)
+
+        Returns:
+            Self for method chaining
+        """
+        self._kwargs["resume"] = enable
+        return self
+
+    def mark_eval_run(self, enable: bool = True) -> "UiPathRuntimeContextBuilder":
+        """Mark this as an evaluation run.
+
+        Args:
+            enable: Whether this is an eval run (defaults to True)
+
+        Returns:
+            Self for method chaining
+        """
+        self._kwargs["is_eval_run"] = enable
+        return self
+
+    def build(self) -> "UiPathRuntimeContext":
+        """Build and return the UiPathRuntimeContext instance.
+
+        Returns:
+            A configured UiPathRuntimeContext instance
+        """
+        config_path = self._kwargs.pop("config_path", None)
+        if config_path:
+            # Create context from config first, then update with any additional kwargs
+            context = UiPathRuntimeContext.from_config(config_path)
+            for key, value in self._kwargs.items():
+                if hasattr(context, key):
+                    setattr(context, key, value)
+            return context
+        else:
+            return UiPathRuntimeContext(**self._kwargs)
+
+
 class UiPathRuntimeContext(BaseModel):
     """Context information passed throughout the runtime execution."""
 
@@ -422,8 +545,8 @@ class UiPathBaseRuntime(ABC):
             content = execution_result.to_dict()
             logger.debug(content)
 
-            # Always write output file at runtime
-            if self.context.job_id:
+            # Always write output file at runtime, except evaluation runs
+            if self.context.job_id and not self.context.is_eval_run:
                 with open(self.output_file_path, "w") as f:
                     json.dump(content, f, indent=2, default=str)
 
@@ -521,6 +644,9 @@ class UiPathRuntimeFactory(Generic[T, C]):
         self.tracer_span_processors: List[SpanProcessor] = []
         trace.set_tracer_provider(self.tracer_provider)
 
+        if os.getenv("UIPATH_JOB_KEY"):
+            self.add_span_exporter(LlmOpsHttpExporter())
+
     def add_span_exporter(
         self,
         span_exporter: SpanExporter,
@@ -591,9 +717,12 @@ class UiPathExecutionTraceProcessorMixin:
             parent_span = trace.get_current_span()
 
         if parent_span and parent_span.is_recording():
-            run_id = parent_span.attributes.get("execution.id")  # type: ignore[attr-defined]
-            if run_id:
-                span.set_attribute("execution.id", run_id)
+            execution_id = parent_span.attributes.get("execution.id")  # type: ignore[attr-defined]
+            if execution_id:
+                span.set_attribute("execution.id", execution_id)
+            evaluation_id = parent_span.attributes.get("evaluation.id")  # type: ignore[attr-defined]
+            if evaluation_id:
+                span.set_attribute("evaluation.id", evaluation_id)
 
 
 class UiPathExecutionBatchTraceProcessor(

uipath/_cli/_utils/_eval_set.py

@@ -0,0 +1,84 @@
+import json
+from pathlib import Path
+from typing import List, Optional
+
+import click
+
+from uipath._cli._evals._models import EvaluationSet
+from uipath._cli._utils._console import ConsoleLogger
+
+console = ConsoleLogger()
+
+
+class EvalHelpers:
+    @staticmethod
+    def auto_discover_eval_set() -> str:
+        """Auto-discover evaluation set from evals/eval-sets directory.
+
+        Returns:
+            Path to the evaluation set file
+
+        Raises:
+            ValueError: If no eval set found or multiple eval sets exist
+        """
+        eval_sets_dir = Path("evals/eval-sets")
+
+        if not eval_sets_dir.exists():
+            raise ValueError(
+                "No 'evals/eval-sets' directory found. "
+                "Please set 'UIPATH_PROJECT_ID' env var and run 'uipath pull'."
+            )
+
+        eval_set_files = list(eval_sets_dir.glob("*.json"))
+
+        if not eval_set_files:
+            raise ValueError(
+                "No evaluation set files found in 'evals/eval-sets' directory. "
+            )
+
+        if len(eval_set_files) > 1:
+            file_names = [f.name for f in eval_set_files]
+            raise ValueError(
+                f"Multiple evaluation sets found: {file_names}. "
+                f"Please specify which evaluation set to use: 'uipath eval [entrypoint] <eval_set_path>'"
+            )
+
+        eval_set_path = str(eval_set_files[0])
+        console.info(
+            f"Auto-discovered evaluation set: {click.style(eval_set_path, fg='cyan')}"
+        )
+
+        eval_set_path_obj = Path(eval_set_path)
+        if not eval_set_path_obj.is_file() or eval_set_path_obj.suffix != ".json":
+            raise ValueError("Evaluation set must be a JSON file")
+
+        return eval_set_path
+
+    @staticmethod
+    def load_eval_set(
+        eval_set_path: str, eval_ids: Optional[List[str]] = None
+    ) -> EvaluationSet:
+        """Load the evaluation set from file.
+
+        Returns:
+            The loaded evaluation set as EvaluationSet model
+        """
+        try:
+            with open(eval_set_path, "r", encoding="utf-8") as f:
+                data = json.load(f)
+        except json.JSONDecodeError as e:
+            raise ValueError(
+                f"Invalid JSON in evaluation set file '{eval_set_path}': {str(e)}. "
+                f"Please check the file for syntax errors."
+            ) from e
+
+        try:
+            eval_set = EvaluationSet(**data)
+        except (TypeError, ValueError) as e:
+            raise ValueError(
+                f"Invalid evaluation set format in '{eval_set_path}': {str(e)}. "
+                f"Please verify the evaluation set structure."
+            ) from e
+        if eval_ids:
+            eval_set.extract_selected_evals(eval_ids)
+        return eval_set

uipath/_cli/cli_auth.py

@@ -1,61 +1,15 @@
-import asyncio
-import json
-import os
-import socket
-import webbrowser
 from typing import Optional
-from urllib.parse import urlparse
 
 import click
 
 from ..telemetry import track
-from ._auth._auth_server import HTTPServer
-from ._auth._client_credentials import ClientCredentialsService
-from ._auth._oidc_utils import get_auth_config, get_auth_url
-from ._auth._portal_service import PortalService, select_tenant
-from ._auth._utils import update_auth_file, update_env_file
+from ._auth._auth_service import AuthService
 from ._utils._common import environment_options
 from ._utils._console import ConsoleLogger
 
 console = ConsoleLogger()
 
 
-def is_port_in_use(port: int) -> bool:
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        try:
-            s.bind(("localhost", port))
-            s.close()
-            return False
-        except socket.error:
-            return True
-
-
-def set_port():
-    auth_config = get_auth_config()
-    port = int(auth_config.get("port", 8104))
-    port_option_one = int(auth_config.get("portOptionOne", 8104))  # type: ignore
-    port_option_two = int(auth_config.get("portOptionTwo", 8055))  # type: ignore
-    port_option_three = int(auth_config.get("portOptionThree", 42042))  # type: ignore
-    if is_port_in_use(port):
-        if is_port_in_use(port_option_one):
-            if is_port_in_use(port_option_two):
-                if is_port_in_use(port_option_three):
-                    console.error(
-                        "All configured ports are in use. Please close applications using ports or configure different ports."
-                    )
-                else:
-                    port = port_option_three
-            else:
-                port = port_option_two
-        else:
-            port = port_option_one
-    auth_config["port"] = port
-    with open(
-        os.path.join(os.path.dirname(__file__), "..", "auth_config.json"), "w"
-    ) as f:
-        json.dump(auth_config, f)
-
-
 @click.command()
 @environment_options
 @click.option(
@@ -63,6 +17,7 @@ def set_port():
     "--force",
     is_flag=True,
     required=False,
+    default=False,
     help="Force new token",
 )
 @click.option(
@@ -89,7 +44,7 @@ def set_port():
 @track
 def auth(
     domain,
-    force: Optional[bool] = False,
+    force: bool = False,
     client_id: Optional[str] = None,
     client_secret: Optional[str] = None,
     base_url: Optional[str] = None,
@@ -108,109 +63,21 @@ def auth(
     - Set REQUESTS_CA_BUNDLE to specify a custom CA bundle for SSL verification
     - Set UIPATH_DISABLE_SSL_VERIFY to disable SSL verification (not recommended)
     """
-    uipath_url = os.getenv("UIPATH_URL")
-    if uipath_url and domain == "cloud":  # "cloud" is the default
-        parsed_url = urlparse(uipath_url)
-        if parsed_url.scheme and parsed_url.netloc:
-            domain = f"{parsed_url.scheme}://{parsed_url.netloc}"
-        else:
-            console.error(
-                f"Malformed UIPATH_URL: '{uipath_url}'. Please ensure it includes both scheme and netloc (e.g., 'https://cloud.uipath.com')."
+    auth_service = AuthService(
+        domain,
+        force=force,
+        client_id=client_id,
+        client_secret=client_secret,
+        base_url=base_url,
+        scope=scope,
+    )
+    with console.spinner("Authenticating with UiPath ..."):
+        try:
+            auth_service.authenticate()
+            console.success(
+                "Authentication successful.",
             )
-            return
-
-    # Check if client credentials are provided for unattended authentication
-    if client_id and client_secret:
-        if not base_url:
+        except KeyboardInterrupt:
             console.error(
-                "--base-url is required when using client credentials authentication."
-            )
-            return
-
-        with console.spinner("Authenticating with client credentials ..."):
-            credentials_service = ClientCredentialsService(domain)
-
-            # If base_url is provided, extract domain from it to override the CLI domain parameter
-            if base_url:
-                extracted_domain = credentials_service.extract_domain_from_base_url(
-                    base_url
-                )
-                credentials_service.domain = extracted_domain
-
-            token_data = credentials_service.authenticate(
-                client_id, client_secret, scope
-            )
-
-            if token_data:
-                credentials_service.setup_environment(token_data, base_url)
-                console.success(
-                    "Client credentials authentication successful.",
-                )
-            else:
-                console.error(
-                    "Client credentials authentication failed. Please check your credentials.",
-                )
-        return
-
-    # Interactive authentication flow (existing logic)
-    with console.spinner("Authenticating with UiPath ..."):
-        with PortalService(domain) as portal_service:
-            if not force:
-                if (
-                    os.getenv("UIPATH_URL")
-                    and os.getenv("UIPATH_TENANT_ID")
-                    and os.getenv("UIPATH_ORGANIZATION_ID")
-                ):
-                    try:
-                        portal_service.ensure_valid_token()
-                        console.success(
-                            "Authentication successful.",
-                        )
-                        return
-                    except Exception:
-                        console.info(
-                            "Authentication token is invalid. Please reauthenticate.",
-                        )
-
-            auth_url, code_verifier, state = get_auth_url(domain)
-
-            webbrowser.open(auth_url, 1)
-            auth_config = get_auth_config()
-
-            console.link(
-                "If a browser window did not open, please open the following URL in your browser:",
-                auth_url,
+                "Authentication cancelled by user.",
             )
-
-            try:
-                server = HTTPServer(port=auth_config["port"])
-                token_data = asyncio.run(server.start(state, code_verifier, domain))
-
-                if not token_data:
-                    console.error(
-                        "Authentication failed. Please try again.",
-                    )
-                    return
-
-                portal_service.update_token_data(token_data)
-                update_auth_file(token_data)
-                access_token = token_data["access_token"]
-                update_env_file({"UIPATH_ACCESS_TOKEN": access_token})
-
-                tenants_and_organizations = (
-                    portal_service.get_tenants_and_organizations()
-                )
-                base_url = select_tenant(domain, tenants_and_organizations)
-                try:
-                    portal_service.post_auth(base_url)
-                    console.success(
-                        "Authentication successful.",
-                    )
-                except Exception:
-                    console.error(
-                        "Could not prepare the environment. Please try again.",
-                    )
-            except KeyboardInterrupt:
-                console.error(
-                    "Authentication cancelled by user.",
-                )