uipath 2.1.42__py3-none-any.whl → 2.1.44__py3-none-any.whl

uipath/_cli/_evals/_models/_agent_execution_output.py

@@ -0,0 +1,14 @@
+from opentelemetry.sdk.trace import ReadableSpan
+from pydantic import BaseModel, ConfigDict
+
+from uipath._cli._runtime._contracts import UiPathRuntimeResult
+
+
+class UiPathEvalRunExecutionOutput(BaseModel):
+    """Result of a single agent response."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    execution_time: float
+    spans: list[ReadableSpan]
+    result: UiPathRuntimeResult

uipath/_cli/_evals/_runtime.py

@@ -0,0 +1,172 @@
+import copy
+from collections import defaultdict
+from time import time
+from typing import Dict, Generic, List, Optional, Sequence, TypeVar
+
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
+
+from uipath.eval._helpers import auto_discover_entrypoint
+
+from .._runtime._contracts import (
+    UiPathBaseRuntime,
+    UiPathRuntimeContext,
+    UiPathRuntimeFactory,
+    UiPathRuntimeResult,
+    UiPathRuntimeStatus,
+)
+from .._utils._eval_set import EvalHelpers
+from ._models import EvaluationItem
+from ._models._agent_execution_output import UiPathEvalRunExecutionOutput
+
+T = TypeVar("T", bound=UiPathBaseRuntime)
+C = TypeVar("C", bound=UiPathRuntimeContext)
+
+
+class ExecutionSpanExporter(SpanExporter):
+    """Custom exporter that stores spans grouped by execution ids."""
+
+    def __init__(self):
+        # { execution_id -> list of spans }
+        self._spans: Dict[str, List[ReadableSpan]] = defaultdict(list)
+
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        for span in spans:
+            if span.attributes is not None:
+                exec_id = span.attributes.get("execution.id")
+                if exec_id is not None and isinstance(exec_id, str):
+                    self._spans[exec_id].append(span)
+
+        return SpanExportResult.SUCCESS
+
+    def get_spans(self, execution_id: str) -> List[ReadableSpan]:
+        """Retrieve spans for a given execution id."""
+        return self._spans.get(execution_id, [])
+
+    def clear(self, execution_id: Optional[str] = None) -> None:
+        """Clear stored spans for one or all executions."""
+        if execution_id:
+            self._spans.pop(execution_id, None)
+        else:
+            self._spans.clear()
+
+    def shutdown(self) -> None:
+        self.clear()
+
+
+class UiPathEvalContext(UiPathRuntimeContext, Generic[C]):
+    """Context used for evaluation runs."""
+
+    runtime_context: C
+    no_report: bool
+    workers: int
+    eval_set: Optional[str] = None
+    eval_ids: Optional[List[str]] = None
+
+    def __init__(
+        self,
+        runtime_context: C,
+        no_report: bool,
+        workers: int,
+        eval_set: Optional[str] = None,
+        eval_ids: Optional[List[str]] = None,
+        **kwargs,
+    ):
+        super().__init__(
+            runtime_context=runtime_context,  # type: ignore
+            no_report=no_report,
+            workers=workers,
+            eval_set=eval_set,
+            eval_ids=eval_ids,
+            **kwargs,
+        )
+        self._auto_discover()
+
+    def _auto_discover(self):
+        self.runtime_context.entrypoint = (
+            self.runtime_context.entrypoint or auto_discover_entrypoint()
+        )
+        self.eval_set = self.eval_set or EvalHelpers.auto_discover_eval_set()
+
+
+class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
+    """Specialized runtime for evaluation runs, with access to the factory."""
+
+    def __init__(
+        self, context: "UiPathEvalContext[C]", factory: "UiPathRuntimeFactory[T, C]"
+    ):
+        super().__init__(context)
+        self.context: "UiPathEvalContext[C]" = context
+        self.factory: UiPathRuntimeFactory[T, C] = factory
+        self.span_exporter: ExecutionSpanExporter = ExecutionSpanExporter()
+        self.factory.add_span_exporter(self.span_exporter)
+
+    @classmethod
+    def from_eval_context(
+        cls,
+        context: "UiPathEvalContext[C]",
+        factory: "UiPathRuntimeFactory[T, C]",
+    ) -> "UiPathEvalRuntime[T, C]":
+        return cls(context, factory)
+
+    async def execute(self) -> Optional[UiPathRuntimeResult]:
+        """Evaluation logic. Can spawn other runtimes through the factory."""
+        if self.context.eval_set is None:
+            raise ValueError("eval_set must be provided for evaluation runs")
+
+        evaluation_set = EvalHelpers.load_eval_set(
+            self.context.eval_set, self.context.eval_ids
+        )
+        execution_output_list: list[UiPathEvalRunExecutionOutput] = []
+        for eval_item in evaluation_set.evaluations:
+            execution_output_list.append(await self.execute_agent(eval_item))
+
+        self.context.result = UiPathRuntimeResult(
+            output={
+                "results": execution_output_list,
+            },
+            status=UiPathRuntimeStatus.SUCCESSFUL,
+            resume=None,
+        )
+
+        return self.context.runtime_context.result
+
+    def _prepare_new_runtime_context(self, eval_item: EvaluationItem) -> C:
+        runtime_context = copy.deepcopy(self.context.runtime_context)
+        runtime_context.execution_id = eval_item.id
+        runtime_context.input_json = eval_item.inputs
+        # here we can pass other values from eval_item: expectedAgentBehavior, simulationInstructions etc.
+        return runtime_context
+
+    # TODO: this would most likely need to be ported to a public AgentEvaluator class
+    async def execute_agent(
+        self, eval_item: EvaluationItem
+    ) -> "UiPathEvalRunExecutionOutput":
+        runtime_context = self._prepare_new_runtime_context(eval_item)
+        start_time = time()
+        result = await self.factory.execute_in_root_span(
+            runtime_context, root_span=eval_item.name
+        )
+        end_time = time()
+        if runtime_context.execution_id is None:
+            raise ValueError("execution_id must be set for eval runs")
+
+        spans = self.span_exporter.get_spans(runtime_context.execution_id)
+        self.span_exporter.clear(runtime_context.execution_id)
+
+        if result is None:
+            raise ValueError("Execution result cannot be None for eval runs")
+
+        return UiPathEvalRunExecutionOutput(
+            execution_time=end_time - start_time,
+            spans=spans,
+            result=result,
+        )
+
+    async def cleanup(self) -> None:
+        """Cleanup runtime resources."""
+        pass
+
+    async def validate(self) -> None:
+        """Cleanup runtime resources."""
+        pass

uipath/_cli/_runtime/_contracts.py

@@ -9,6 +9,7 @@ from abc import ABC, abstractmethod
 from enum import Enum
 from functools import cached_property
 from typing import Any, Callable, Dict, Generic, List, Optional, Type, TypeVar, Union
+from uuid import uuid4
 
 from opentelemetry import context as context_api
 from opentelemetry import trace
@@ -23,7 +24,7 @@ from opentelemetry.trace import Tracer
 from pydantic import BaseModel, Field
 
 from uipath.agent.conversation import UiPathConversationEvent, UiPathConversationMessage
-from uipath.tracing import TracingManager
+from uipath.tracing import LlmOpsHttpExporter, TracingManager
 
 from ._logging import LogsInterceptor
 
@@ -161,6 +162,128 @@ class UiPathTraceContext(BaseModel):
     reference_id: Optional[str] = None
 
 
+class UiPathRuntimeContextBuilder:
+    """Builder class for UiPathRuntimeContext following the builder pattern."""
+
+    def __init__(self):
+        self._kwargs = {}
+
+    def with_defaults(
+        self, config_path: Optional[str] = None, **kwargs
+    ) -> "UiPathRuntimeContextBuilder":
+        """Apply default configuration similar to UiPathRuntimeContext.with_defaults().
+
+        Args:
+            config_path: Path to the configuration file (defaults to UIPATH_CONFIG_PATH env var or "uipath.json")
+            **kwargs: Additional keyword arguments to pass to with_defaults
+
+        Returns:
+            Self for method chaining
+        """
+        from os import environ as env
+
+        resolved_config_path = config_path or env.get(
+            "UIPATH_CONFIG_PATH", "uipath.json"
+        )
+        self._kwargs["config_path"] = resolved_config_path
+
+        self._kwargs.update(
+            {
+                "job_id": env.get("UIPATH_JOB_KEY"),
+                "trace_id": env.get("UIPATH_TRACE_ID"),
+                "tracing_enabled": env.get("UIPATH_TRACING_ENABLED", True),
+                "logs_min_level": env.get("LOG_LEVEL", "INFO"),
+                **kwargs,  # Allow overriding defaults with provided kwargs
+            }
+        )
+
+        self._kwargs["trace_context"] = UiPathTraceContext(
+            trace_id=env.get("UIPATH_TRACE_ID"),
+            parent_span_id=env.get("UIPATH_PARENT_SPAN_ID"),
+            root_span_id=env.get("UIPATH_ROOT_SPAN_ID"),
+            enabled=env.get("UIPATH_TRACING_ENABLED", True),
+            job_id=env.get("UIPATH_JOB_KEY"),
+            org_id=env.get("UIPATH_ORGANIZATION_ID"),
+            tenant_id=env.get("UIPATH_TENANT_ID"),
+            process_key=env.get("UIPATH_PROCESS_UUID"),
+            folder_key=env.get("UIPATH_FOLDER_KEY"),
+            reference_id=env.get("UIPATH_JOB_KEY") or str(uuid4()),
+        )
+
+        return self
+
+    def with_entrypoint(self, entrypoint: str) -> "UiPathRuntimeContextBuilder":
+        """Set the entrypoint for the runtime context.
+
+        Args:
+            entrypoint: The entrypoint to execute
+
+        Returns:
+            Self for method chaining
+        """
+        self._kwargs["entrypoint"] = entrypoint
+        return self
+
+    def with_input(
+        self, input_data: Optional[str] = None, input_file: Optional[str] = None
+    ) -> "UiPathRuntimeContextBuilder":
+        """Set the input data for the runtime context.
+
+        Args:
+            input_data: The input data as a string
+            input_file: Path to the input file
+
+        Returns:
+            Self for method chaining
+        """
+        if input_data is not None:
+            self._kwargs["input"] = input_data
+        if input_file is not None:
+            self._kwargs["input_file"] = input_file
+        return self
+
+    def with_resume(self, enable: bool = True) -> "UiPathRuntimeContextBuilder":
+        """Enable or disable resume mode for the runtime context.
+
+        Args:
+            enable: Whether to enable resume mode (defaults to True)
+
+        Returns:
+            Self for method chaining
+        """
+        self._kwargs["resume"] = enable
+        return self
+
+    def mark_eval_run(self, enable: bool = True) -> "UiPathRuntimeContextBuilder":
+        """Mark this as an evaluation run.
+
+        Args:
+            enable: Whether this is an eval run (defaults to True)
+
+        Returns:
+            Self for method chaining
+        """
+        self._kwargs["is_eval_run"] = enable
+        return self
+
+    def build(self) -> "UiPathRuntimeContext":
+        """Build and return the UiPathRuntimeContext instance.
+
+        Returns:
+            A configured UiPathRuntimeContext instance
+        """
+        config_path = self._kwargs.pop("config_path", None)
+        if config_path:
+            # Create context from config first, then update with any additional kwargs
+            context = UiPathRuntimeContext.from_config(config_path)
+            for key, value in self._kwargs.items():
+                if hasattr(context, key):
+                    setattr(context, key, value)
+            return context
+        else:
+            return UiPathRuntimeContext(**self._kwargs)
+
+
 class UiPathRuntimeContext(BaseModel):
     """Context information passed throughout the runtime execution."""
 
@@ -422,8 +545,8 @@ class UiPathBaseRuntime(ABC):
             content = execution_result.to_dict()
             logger.debug(content)
 
-            # Always write output file at runtime
-            if self.context.job_id:
+            # Always write output file at runtime, except evaluation runs
+            if self.context.job_id and not self.context.is_eval_run:
                 with open(self.output_file_path, "w") as f:
                     json.dump(content, f, indent=2, default=str)
 
@@ -521,6 +644,9 @@ class UiPathRuntimeFactory(Generic[T, C]):
         self.tracer_span_processors: List[SpanProcessor] = []
         trace.set_tracer_provider(self.tracer_provider)
 
+        if os.getenv("UIPATH_JOB_KEY"):
+            self.add_span_exporter(LlmOpsHttpExporter())
+
     def add_span_exporter(
         self,
         span_exporter: SpanExporter,
@@ -591,9 +717,12 @@ class UiPathExecutionTraceProcessorMixin:
             parent_span = trace.get_current_span()
 
         if parent_span and parent_span.is_recording():
-            run_id = parent_span.attributes.get("execution.id")  # type: ignore[attr-defined]
-            if run_id:
-                span.set_attribute("execution.id", run_id)
+            execution_id = parent_span.attributes.get("execution.id")  # type: ignore[attr-defined]
+            if execution_id:
+                span.set_attribute("execution.id", execution_id)
+            evaluation_id = parent_span.attributes.get("evaluation.id")  # type: ignore[attr-defined]
+            if evaluation_id:
+                span.set_attribute("evaluation.id", evaluation_id)
 
 
 class UiPathExecutionBatchTraceProcessor(

uipath/_cli/_utils/_eval_set.py

@@ -0,0 +1,84 @@
+import json
+from pathlib import Path
+from typing import List, Optional
+
+import click
+
+from uipath._cli._evals._models import EvaluationSet
+from uipath._cli._utils._console import ConsoleLogger
+
+console = ConsoleLogger()
+
+
+class EvalHelpers:
+    @staticmethod
+    def auto_discover_eval_set() -> str:
+        """Auto-discover evaluation set from evals/eval-sets directory.
+
+        Returns:
+            Path to the evaluation set file
+
+        Raises:
+            ValueError: If no eval set found or multiple eval sets exist
+        """
+        eval_sets_dir = Path("evals/eval-sets")
+
+        if not eval_sets_dir.exists():
+            raise ValueError(
+                "No 'evals/eval-sets' directory found. "
+                "Please set 'UIPATH_PROJECT_ID' env var and run 'uipath pull'."
+            )
+
+        eval_set_files = list(eval_sets_dir.glob("*.json"))
+
+        if not eval_set_files:
+            raise ValueError(
+                "No evaluation set files found in 'evals/eval-sets' directory. "
+            )
+
+        if len(eval_set_files) > 1:
+            file_names = [f.name for f in eval_set_files]
+            raise ValueError(
+                f"Multiple evaluation sets found: {file_names}. "
+                f"Please specify which evaluation set to use: 'uipath eval [entrypoint] <eval_set_path>'"
+            )
+
+        eval_set_path = str(eval_set_files[0])
+        console.info(
+            f"Auto-discovered evaluation set: {click.style(eval_set_path, fg='cyan')}"
+        )
+
+        eval_set_path_obj = Path(eval_set_path)
+        if not eval_set_path_obj.is_file() or eval_set_path_obj.suffix != ".json":
+            raise ValueError("Evaluation set must be a JSON file")
+
+        return eval_set_path
+
+    @staticmethod
+    def load_eval_set(
+        eval_set_path: str, eval_ids: Optional[List[str]] = None
+    ) -> EvaluationSet:
+        """Load the evaluation set from file.
+
+        Returns:
+            The loaded evaluation set as EvaluationSet model
+        """
+        try:
+            with open(eval_set_path, "r", encoding="utf-8") as f:
+                data = json.load(f)
+        except json.JSONDecodeError as e:
+            raise ValueError(
+                f"Invalid JSON in evaluation set file '{eval_set_path}': {str(e)}. "
+                f"Please check the file for syntax errors."
+            ) from e
+
+        try:
+            eval_set = EvaluationSet(**data)
+        except (TypeError, ValueError) as e:
+            raise ValueError(
+                f"Invalid evaluation set format in '{eval_set_path}': {str(e)}. "
+                f"Please verify the evaluation set structure."
+            ) from e
+        if eval_ids:
+            eval_set.extract_selected_evals(eval_ids)
+        return eval_set

uipath/_cli/cli_eval.py

@@ -2,13 +2,22 @@
 import ast
 import asyncio
 import os
-from typing import List, Optional, Tuple
+from datetime import datetime, timezone
+from typing import List, Optional
 
 import click
 
+from uipath._cli._evals._runtime import UiPathEvalContext, UiPathEvalRuntime
+from uipath._cli._runtime._contracts import (
+    UiPathRuntimeContext,
+    UiPathRuntimeContextBuilder,
+    UiPathRuntimeFactory,
+)
+from uipath._cli._runtime._runtime import UiPathRuntime
+from uipath._cli.middlewares import MiddlewareResult, Middlewares
+
 from .._utils.constants import ENV_JOB_ID
 from ..telemetry import track
-from ._evals.evaluation_service import EvaluationService
 from ._utils._console import ConsoleLogger
 
 console = ConsoleLogger()
@@ -22,48 +31,60 @@ class LiteralOption(click.Option):
             raise click.BadParameter(value) from e
 
 
-def eval_agent(
+def eval_agent_middleware(
     entrypoint: Optional[str] = None,
     eval_set: Optional[str] = None,
     eval_ids: Optional[List[str]] = None,
     workers: int = 8,
     no_report: bool = False,
     **kwargs,
-) -> Tuple[bool, Optional[str], Optional[str]]:
-    """Core evaluation logic that can be called programmatically.
-
-    Args:
-        entrypoint: Path to the agent script to evaluate (optional, will auto-discover if not provided)
-        eval_set: Path to the evaluation set JSON file (optional, will auto-discover if not provided)
-        eval_ids: Optional list of evaluation IDs
-        workers: Number of parallel workers for running evaluations
-        no_report: Do not report the evaluation results
-        **kwargs: Additional arguments for future extensibility
+) -> MiddlewareResult:
+    def generate_eval_context(
+        runtime_context: UiPathRuntimeContext,
+    ) -> UiPathEvalContext:
+        os.makedirs("evals/results", exist_ok=True)
+        timestamp = datetime.now(timezone.utc).strftime("%M-%H-%d-%m-%Y")
+        base_context = UiPathRuntimeContextBuilder().with_defaults().build()
+        # TODO: the name should include the eval_set name. those files should not be commited to SW
+        base_context.execution_output_file = (
+            f"evals/results/{timestamp}.json"
+            if not os.getenv("UIPATH_JOB_KEY")
+            else None
+        )
+        return UiPathEvalContext(
+            runtime_context=runtime_context,
+            no_report=no_report,
+            workers=workers,
+            eval_set=eval_set,
+            eval_ids=eval_ids,
+            **kwargs,
+            **base_context.model_dump(),
+        )
 
-    Returns:
-        Tuple containing:
-            - success: True if evaluation was successful
-            - error_message: Error message if any
-            - info_message: Info message if any
-    """
     try:
-        if workers < 1:
-            return False, "Number of workers must be at least 1", None
-
-        print("EVAL SET")
-        print(eval_set)
-        if eval_set is not None and len(eval_set) == 0:
-            return False, "Evaluation set must not be empty", None
-
-        service = EvaluationService(
-            entrypoint, eval_set, eval_ids, workers, report_progress=not no_report
+        runtime_factory = UiPathRuntimeFactory(UiPathRuntime, UiPathRuntimeContext)
+        context = (
+            UiPathRuntimeContextBuilder()
+            .with_defaults(**kwargs)
+            .with_entrypoint(entrypoint)
+            .with_entrypoint(entrypoint)
+            .mark_eval_run()
+            .build()
         )
-        asyncio.run(service.run_evaluation())
 
-        return True, None, "Evaluation completed successfully"
+        async def execute():
+            async with UiPathEvalRuntime.from_eval_context(
+                factory=runtime_factory, context=generate_eval_context(context)
+            ) as eval_runtime:
+                await eval_runtime.execute()
+
+        asyncio.run(execute())
+        return MiddlewareResult(should_continue=False)
 
     except Exception as e:
-        return False, f"Error running evaluation: {str(e)}", None
+        return MiddlewareResult(
+            should_continue=False, error_message=f"Error running evaluation: {str(e)}"
+        )
 
 
 @click.command()
@@ -99,19 +120,29 @@ def eval(
         workers: Number of parallel workers for running evaluations
         no_report: Do not report the evaluation results
     """
-    success, error_message, info_message = eval_agent(
-        entrypoint=entrypoint,
-        eval_set=eval_set,
-        eval_ids=eval_ids,
-        workers=workers,
+    result = Middlewares.next(
+        "eval",
+        entrypoint,
+        eval_set,
+        eval_ids,
         no_report=no_report,
+        workers=workers,
     )
-    if error_message:
-        console.error(error_message)
-        click.get_current_context().exit(1)
 
-    if info_message:
-        console.success(info_message)
+    if result.should_continue:
+        result = eval_agent_middleware(
+            entrypoint=entrypoint,
+            eval_set=eval_set,
+            eval_ids=eval_ids,
+            workers=workers,
+            no_report=no_report,
+        )
+    if result.should_continue:
+        console.error("Could not process the request with any available handler.")
+    if result.error_message:
+        console.error(result.error_message)
+
+    console.success("Evaluation completed successfully")
 
 
 if __name__ == "__main__":

uipath/_cli/cli_run.py

@@ -4,12 +4,10 @@ import os
 import traceback
 from os import environ as env
 from typing import Optional, Tuple
-from uuid import uuid4
 
 import click
 
 from uipath._cli._utils._debug import setup_debugging
-from uipath.tracing import LlmOpsHttpExporter
 
 from .._utils.constants import (
     ENV_JOB_ID,
@@ -17,9 +15,9 @@ from .._utils.constants import (
 from ..telemetry import track
 from ._runtime._contracts import (
     UiPathRuntimeContext,
+    UiPathRuntimeContextBuilder,
     UiPathRuntimeError,
     UiPathRuntimeFactory,
-    UiPathTraceContext,
 )
 from ._runtime._runtime import UiPathRuntime
 from ._utils._console import ConsoleLogger
@@ -61,41 +59,17 @@ Usage: `uipath run <entrypoint_path> <input_arguments> [-f <input_json_file_path
         )
 
     try:
-        context = UiPathRuntimeContext.from_config(
-            env.get("UIPATH_CONFIG_PATH", "uipath.json"), **kwargs
-        )
-        context.entrypoint = entrypoint
-        context.input = input
-        context.resume = resume
-        context.job_id = env.get("UIPATH_JOB_KEY")
-        context.trace_id = env.get("UIPATH_TRACE_ID")
-        context.input_file = kwargs.get("input_file", None)
-        context.execution_output_file = kwargs.get("execution_output_file", None)
-        context.is_eval_run = kwargs.get("is_eval_run", False)
-        context.logs_min_level = env.get("LOG_LEVEL", "INFO")
-        context.tracing_enabled = env.get("UIPATH_TRACING_ENABLED", True)
-        context.trace_context = UiPathTraceContext(
-            trace_id=env.get("UIPATH_TRACE_ID"),
-            parent_span_id=env.get("UIPATH_PARENT_SPAN_ID"),
-            root_span_id=env.get("UIPATH_ROOT_SPAN_ID"),
-            enabled=env.get("UIPATH_TRACING_ENABLED", True),
-            job_id=env.get("UIPATH_JOB_KEY"),
-            org_id=env.get("UIPATH_ORGANIZATION_ID"),
-            tenant_id=env.get("UIPATH_TENANT_ID"),
-            process_key=env.get("UIPATH_PROCESS_UUID"),
-            folder_key=env.get("UIPATH_FOLDER_KEY"),
-            reference_id=env.get("UIPATH_JOB_KEY") or str(uuid4()),
-        )
-
         runtime_factory = UiPathRuntimeFactory(UiPathRuntime, UiPathRuntimeContext)
+        context = (
+            UiPathRuntimeContextBuilder()
+            .with_defaults(**kwargs)
+            .with_entrypoint(entrypoint)
+            .with_input(input, input_file=kwargs.get("input_file"))
+            .with_resume(resume)
+            .build()
+        )
 
-        if context.job_id:
-            runtime_factory.add_span_exporter(LlmOpsHttpExporter())
-
-        async def execute():
-            await runtime_factory.execute(context)
-
-        asyncio.run(execute())
+        asyncio.run(runtime_factory.execute(context))
 
         return MiddlewareResult(should_continue=False)
 

uipath/_cli/middlewares.py

@@ -30,6 +30,7 @@ class Middlewares:
         "run": [],
         "dev": [],
         "invoke": [],
+        "eval": [],
     }
     _plugins_loaded = False