uipath 2.1.74__tar.gz → 2.1.76__tar.gz

{uipath-2.1.74 → uipath-2.1.76}/CONTRIBUTING.md

@@ -21,7 +21,21 @@
     uv venv
     ```
 
-4. **Install dependencies**:
+4. **Activate the virtual environment**:
+    - Linux/Mac
+    ```sh
+    source .venv/bin/activate
+    ```
+    - Windows Powershell
+    ```sh
+    .venv\Scripts\Activate.ps1
+    ```
+    - Windows Bash
+    ```sh
+    source .venv/Scripts/activate
+    ```
+
+5. **Install dependencies**:
     ```sh
     uv sync --all-extras --no-cache
     ```

{uipath-2.1.74 → uipath-2.1.76}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: uipath
-Version: 2.1.74
+Version: 2.1.76
 Summary: Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools.
 Project-URL: Homepage, https://uipath.com
 Project-URL: Repository, https://github.com/UiPath/uipath-python

{uipath-2.1.74 → uipath-2.1.76}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.1.74"
+version = "2.1.76"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.10"

{uipath-2.1.74 → uipath-2.1.76}/samples/calculator/evals/eval-sets/default.json

@@ -4,7 +4,8 @@
   "name": "Basic Calculator Evaluation Set",
   "batchSize": 10,
   "evaluatorRefs": [
-    "equality"
+    "equality",
+    "llm-as-a-judge"
   ],
   "evaluations": [
     {

uipath-2.1.76/samples/calculator/evals/evaluators/llm-as-a-judge.json

@@ -0,0 +1,13 @@
+{
+  "fileName": "llm-as-a-judge.json",
+  "id": "llm-as-a-judge",
+  "name": "LLMAsAJudge Evaluator",
+  "description": "An evaluator that judges the agent based on it's run history and expected behavior",
+  "category": 3,
+  "type": 7,
+  "prompt": "As an expert evaluator, determine how well the agent did on a scale of 0-100. Focus on if the simulation was successful and if the agent behaved according to the expected output accounting for alternative valid expressions, and reasonable variations in language while maintaining high standards for accuracy and completeness. Provide your score with a justification, explaining briefly and concisely why you gave that score.\n----\nUserOrSyntheticInputGivenToAgent:\n{{UserOrSyntheticInput}}\n----\nSimulationInstructions:\n{{SimulationInstructions}}\n----\nExpectedAgentBehavior:\n{{ExpectedAgentBehavior}}\n----\nAgentRunHistory:\n{{AgentRunHistory}}\n",
+  "targetOutputKey": "*",
+  "model":"gpt-4.1-2025-04-14",
+  "createdAt": "2025-08-21T15:12:58.695Z",
+  "updatedAt": "2025-08-21T15:12:58.695Z"
+}

uipath-2.1.76/src/uipath/_cli/_evals/_console_progress_reporter.py

@@ -0,0 +1,241 @@
+"""Console progress reporter for evaluation runs with line-by-line output."""
+
+import logging
+from typing import Any, Dict
+
+from rich.console import Console
+from rich.rule import Rule
+from rich.table import Table
+
+from uipath._events._event_bus import EventBus
+from uipath._events._events import (
+    EvalRunCreatedEvent,
+    EvalRunUpdatedEvent,
+    EvalSetRunCreatedEvent,
+    EvalSetRunUpdatedEvent,
+    EvaluationEvents,
+)
+from uipath.eval.evaluators import BaseEvaluator
+from uipath.eval.models import ScoreType
+
+logger = logging.getLogger(__name__)
+
+
+class ConsoleProgressReporter:
+    """Handles displaying evaluation progress to the console with line-by-line output."""
+
+    def __init__(self):
+        self.console = Console()
+        self.evaluators: Dict[str, BaseEvaluator[Any]] = {}
+        self.display_started = False
+        self.eval_results_by_name: Dict[str, list[Any]] = {}
+
+    def _convert_score_to_numeric(self, eval_result) -> float:
+        """Convert evaluation result score to numeric value."""
+        score_value = eval_result.result.score
+        if eval_result.result.score_type == ScoreType.BOOLEAN:
+            score_value = 100 if score_value else 0
+        return score_value
+
+    def _get_evaluator_name(self, evaluator_id: str) -> str:
+        """Get evaluator name from ID, with fallback."""
+        return self.evaluators.get(
+            evaluator_id,
+            type(
+                "obj",
+                (object,),
+                {"name": f"Evaluator {evaluator_id[:8]}"},
+            )(),
+        ).name
+
+    def _display_successful_evaluation(self, eval_name: str, eval_results) -> None:
+        """Display results for a successful evaluation."""
+        from rich.text import Text
+
+        if eval_results:
+            result = Text()
+            result.append("▌", style="bold green")
+            result.append(" ", style="")
+            result.append(eval_name, style="bold green")
+            self.console.print(result)
+            table = Table(show_header=False, box=None, padding=(0, 2, 0, 2))
+
+            for eval_result in eval_results:
+                evaluator_name = self._get_evaluator_name(eval_result.evaluator_id)
+                score_value = self._convert_score_to_numeric(eval_result)
+                table.add_row(
+                    f"{evaluator_name}", f"[bold cyan]{score_value:.1f}[/bold cyan]"
+                )
+
+            self.console.print(table)
+        else:
+            result = Text()
+            result.append("  ✓ ", style="bold green")
+            result.append(eval_name, style="bold white")
+            result.append(" - No evaluators", style="dim")
+            self.console.print(result)
+
+    def _extract_error_message(self, payload: EvalRunUpdatedEvent) -> str:
+        return str(payload.exception_details.exception) or "Execution failed"  # type: ignore
+
+    def _display_failed_evaluation(self, eval_name: str) -> None:
+        """Display results for a failed evaluation."""
+        from rich.text import Text
+
+        result = Text()
+        result.append("  ✗ ", style="bold red")
+        result.append(eval_name, style="bold white")
+        self.console.print(result)
+
+    def start_display(self):
+        """Start the display."""
+        if not self.display_started:
+            self.console.print()
+            self.console.print("→ [bold]Running Evaluations[/bold]")
+            self.console.print()
+            self.display_started = True
+
+    async def handle_create_eval_set_run(self, payload: EvalSetRunCreatedEvent) -> None:
+        """Handle evaluation set run creation."""
+        try:
+            self.evaluators = {eval.id: eval for eval in payload.evaluators}
+        except Exception as e:
+            logger.error(f"Failed to handle create eval set run event: {e}")
+
+    async def handle_create_eval_run(self, payload: EvalRunCreatedEvent) -> None:
+        """Handle individual evaluation run creation."""
+        try:
+            if not self.display_started:
+                self.start_display()
+
+            self.console.print(f"  ○ [dim]{payload.eval_item.name}[/dim] - Running...")
+        except Exception as e:
+            logger.error(f"Failed to handle create eval run event: {e}")
+
+    def _display_logs_panel(self, eval_name: str, logs, error_msg: str = "") -> None:
+        """Display execution logs panel with optional exception at the end."""
+        self.console.print(
+            Rule(
+                f"[dim italic]Execution Logs: {eval_name}[/dim italic]",
+                style="dim",
+                align="center",
+            )
+        )
+
+        if logs:
+            for record in logs:
+                self.console.print(f"  [dim]{record.getMessage()}[/dim]")
+        elif not error_msg:
+            self.console.print("  [dim italic]No execution logs[/dim italic]")
+
+        if error_msg:
+            self.console.print(f"  [red]{error_msg}[/red]")
+
+        self.console.print(Rule(style="dim"))
+
+    async def handle_update_eval_run(self, payload: EvalRunUpdatedEvent) -> None:
+        """Handle evaluation run updates."""
+        try:
+            if payload.success:
+                self.eval_results_by_name[payload.eval_item.name] = payload.eval_results
+                self._display_successful_evaluation(
+                    payload.eval_item.name, payload.eval_results
+                )
+                self._display_logs_panel(payload.eval_item.name, payload.logs)
+            else:
+                error_msg = self._extract_error_message(payload)
+                self._display_failed_evaluation(payload.eval_item.name)
+
+                if payload.exception_details.runtime_exception:  # type: ignore
+                    self._display_logs_panel(
+                        payload.eval_item.name, payload.logs, error_msg
+                    )
+                else:
+                    self.console.print(f"  [red]{error_msg}[/red]")
+                    self.console.print()
+        except Exception as e:
+            logger.error(f"Console reporter error: {e}")
+
+    async def handle_update_eval_set_run(self, payload: EvalSetRunUpdatedEvent) -> None:
+        """Handle evaluation set run completion."""
+        try:
+            self.final_results = payload.evaluator_scores
+            self.display_final_results()
+        except Exception as e:
+            logger.error(f"Console reporter error: {e}")
+
+    def display_final_results(self):
+        """Display final results summary."""
+        self.console.print()
+
+        if hasattr(self, "final_results") and self.final_results:
+            from rich.table import Table
+
+            # Group evaluators by ID to organize display
+            evaluator_ids = list(self.final_results.keys())
+
+            # Print title
+            self.console.print("[bold]Evaluation Results[/bold]")
+            self.console.print()
+
+            # Create single summary table
+            summary_table = Table(show_header=True, padding=(0, 2))
+            summary_table.add_column("Evaluation", style="cyan")
+
+            # Add column for each evaluator
+            for evaluator_id in evaluator_ids:
+                evaluator_name = self._get_evaluator_name(evaluator_id)
+                summary_table.add_column(evaluator_name, justify="right")
+
+            # Add row for each evaluation
+            for eval_name, eval_results in self.eval_results_by_name.items():
+                row_values = [eval_name]
+
+                # Get score for each evaluator
+                for evaluator_id in evaluator_ids:
+                    score_found = False
+                    for eval_result in eval_results:
+                        if eval_result.evaluator_id == evaluator_id:
+                            score_value = self._convert_score_to_numeric(eval_result)
+                            row_values.append(f"{score_value:.1f}")
+                            score_found = True
+                            break
+
+                    if not score_found:
+                        row_values.append("-")
+
+                summary_table.add_row(*row_values)
+
+            # Add separator row before average
+            summary_table.add_section()
+
+            # Add average row
+            avg_row_values = ["[bold]Average[/bold]"]
+            for evaluator_id in evaluator_ids:
+                avg_score = self.final_results[evaluator_id]
+                avg_row_values.append(f"[bold]{avg_score:.1f}[/bold]")
+
+            summary_table.add_row(*avg_row_values)
+
+            self.console.print(summary_table)
+            self.console.print()
+        else:
+            self.console.print(
+                "→ [bold green]All evaluations completed successfully![/bold green]"
+            )
+            self.console.print()
+
+    async def subscribe_to_eval_runtime_events(self, event_bus: EventBus) -> None:
+        """Subscribe to evaluation runtime events."""
+        event_bus.subscribe(
+            EvaluationEvents.CREATE_EVAL_SET_RUN, self.handle_create_eval_set_run
+        )
+        event_bus.subscribe(
+            EvaluationEvents.CREATE_EVAL_RUN, self.handle_create_eval_run
+        )
+        event_bus.subscribe(
+            EvaluationEvents.UPDATE_EVAL_RUN, self.handle_update_eval_run
+        )
+        event_bus.subscribe(
+            EvaluationEvents.UPDATE_EVAL_SET_RUN, self.handle_update_eval_set_run
+        )

uipath-2.1.76/src/uipath/_cli/_evals/_models/_exceptions.py

@@ -0,0 +1,5 @@
+class EvaluationRuntimeException(Exception):
+    def __init__(self, spans, logs, root_exception):
+        self.spans = spans
+        self.logs = logs
+        self.root_exception = root_exception

{uipath-2.1.74 → uipath-2.1.76}/src/uipath/_cli/_evals/_models/_output.py

@@ -1,3 +1,4 @@
+import logging
 from typing import List, Optional
 
 from opentelemetry.sdk.trace import ReadableSpan
@@ -15,6 +16,7 @@ class UiPathEvalRunExecutionOutput(BaseModel):
 
     execution_time: float
     spans: list[ReadableSpan]
+    logs: list[logging.LogRecord]
     result: UiPathRuntimeResult
 
 

{uipath-2.1.74 → uipath-2.1.76}/src/uipath/_cli/_evals/_progress_reporter.py

@@ -7,6 +7,7 @@ import os
 from typing import Any, Dict, List
 
 from opentelemetry import trace
+from rich.console import Console
 
 from uipath import UiPath
 from uipath._cli._evals._models._evaluation_set import EvaluationItem, EvaluationStatus
@@ -68,6 +69,7 @@ class StudioWebProgressReporter:
 
         self._client = uipath.api_client
         self._console = console_logger
+        self._rich_console = Console()
         self._project_id = os.getenv("UIPATH_PROJECT_ID", None)
         if not self._project_id:
             logger.warning(
@@ -79,6 +81,10 @@ class StudioWebProgressReporter:
         self.evaluator_scores: Dict[str, List[float]] = {}
         self.eval_run_ids: Dict[str, str] = {}
 
+    def _format_error_message(self, error: Exception, context: str) -> None:
+        """Helper method to format and display error messages consistently."""
+        self._rich_console.print(f"    • \u26a0  [dim]{context}: {error}[/dim]")
+
     @gracefully_handle_errors
     async def create_eval_set_run(
         self,
@@ -182,7 +188,7 @@ class StudioWebProgressReporter:
             logger.debug(f"Created eval set run with ID: {eval_set_run_id}")
 
         except Exception as e:
-            logger.error(f"Failed to handle create eval set run event: {e}")
+            self._format_error_message(e, "StudioWeb create eval set run error")
 
     async def handle_create_eval_run(self, payload: EvalRunCreatedEvent) -> None:
         try:
@@ -197,7 +203,7 @@ class StudioWebProgressReporter:
                 logger.warning("Cannot create eval run: eval_set_run_id not available")
 
         except Exception as e:
-            logger.error(f"Failed to handle create eval run event: {e}")
+            self._format_error_message(e, "StudioWeb create eval run error")
 
     async def handle_update_eval_run(self, payload: EvalRunUpdatedEvent) -> None:
         try:
@@ -238,7 +244,7 @@ class StudioWebProgressReporter:
                 logger.debug(f"Updated eval run with ID: {eval_run_id}")
 
         except Exception as e:
-            logger.error(f"Failed to handle update eval run event: {e}")
+            self._format_error_message(e, "StudioWeb reporting error")
 
     async def handle_update_eval_set_run(self, payload: EvalSetRunUpdatedEvent) -> None:
         try:
@@ -254,7 +260,7 @@ class StudioWebProgressReporter:
                 )
 
         except Exception as e:
-            logger.error(f"Failed to handle update eval set run event: {e}")
+            self._format_error_message(e, "StudioWeb update eval set run error")
 
     async def subscribe_to_eval_runtime_events(self, event_bus: EventBus) -> None:
         event_bus.subscribe(
@@ -270,7 +276,7 @@ class StudioWebProgressReporter:
             EvaluationEvents.UPDATE_EVAL_SET_RUN, self.handle_update_eval_set_run
         )
 
-        logger.info("StudioWeb progress reporter subscribed to evaluation events")
+        logger.debug("StudioWeb progress reporter subscribed to evaluation events")
 
     def _extract_agent_snapshot(self, entrypoint: str) -> StudioWebAgentSnapshot:
         try: