uipath 2.1.7__py3-none-any.whl → 2.1.9__py3-none-any.whl

uipath/_cli/__init__.py

@@ -5,6 +5,7 @@ import click
 
 from .cli_auth import auth as auth  # type: ignore
 from .cli_deploy import deploy as deploy  # type: ignore
+from .cli_eval import eval as eval  # type: ignore
 from .cli_init import init as init  # type: ignore
 from .cli_invoke import invoke as invoke  # type: ignore
 from .cli_new import new as new  # type: ignore
@@ -67,3 +68,4 @@ cli.add_command(auth)
 cli.add_command(invoke)
 cli.add_command(push)
 cli.add_command(pull)
+cli.add_command(eval)

uipath/_cli/_auth/auth_config.json

@@ -1,6 +1,6 @@
 {
   "client_id": "36dea5b8-e8bb-423d-8e7b-c808df8f1c00",
   "redirect_uri": "http://localhost:__PY_REPLACE_PORT__/oidc/login",
-  "scope": "offline_access OrchestratorApiUserAccess StudioWebBackend IdentityServerApi ConnectionService DataService DocumentUnderstanding EnterpriseContextService Directory JamJamApi LLMGateway LLMOps OMS RCS.FolderAuthorization TM.Projects TM.TestCases TM.Requirements TM.TestSets",
+  "scope": "offline_access ProcessMining OrchestratorApiUserAccess StudioWebBackend IdentityServerApi ConnectionService DataService DocumentUnderstanding EnterpriseContextService Directory JamJamApi LLMGateway LLMOps OMS RCS.FolderAuthorization TM.Projects TM.TestCases TM.Requirements TM.TestSets",
   "port": 8104
 }

uipath/_cli/_evals/_evaluators/__init__.py

@@ -0,0 +1,20 @@
+"""Evaluators package for the evaluation system.
+
+This package contains all evaluator types and the factory for creating them.
+"""
+
+from ._agent_scorer_evaluator import AgentScorerEvaluator
+from ._deterministic_evaluator import DeterministicEvaluator
+from ._evaluator_base import EvaluatorBase
+from ._evaluator_factory import EvaluatorFactory
+from ._llm_as_judge_evaluator import LlmAsAJudgeEvaluator
+from ._trajectory_evaluator import TrajectoryEvaluator
+
+__all__ = [
+    "EvaluatorBase",
+    "EvaluatorFactory",
+    "DeterministicEvaluator",
+    "LlmAsAJudgeEvaluator",
+    "AgentScorerEvaluator",
+    "TrajectoryEvaluator",
+]

uipath/_cli/_evals/_evaluators/_agent_scorer_evaluator.py

@@ -0,0 +1,48 @@
+from typing import Any, Dict
+
+from .._models import EvaluationResult
+from ._evaluator_base import EvaluatorBase
+
+
+class AgentScorerEvaluator(EvaluatorBase):
+    """Evaluator that uses an agent to score outputs."""
+
+    def __init__(
+        self,
+        agent_config: Dict[str, Any],
+        scoring_criteria: Dict[str, Any],
+        target_output_key: str = "*",
+    ):
+        """Initialize the agent scorer evaluator.
+
+        Args:
+            agent_config: Configuration for the scoring agent
+            scoring_criteria: Criteria used for scoring
+            target_output_key: Key in output to evaluate ("*" for entire output)
+        """
+        super().__init__()
+        self.agent_config = agent_config or {}
+        self.scoring_criteria = scoring_criteria or {}
+        self.target_output_key = target_output_key
+
+    async def evaluate(
+        self,
+        evaluation_id: str,
+        evaluation_name: str,
+        input_data: Dict[str, Any],
+        expected_output: Dict[str, Any],
+        actual_output: Dict[str, Any],
+    ) -> EvaluationResult:
+        """Evaluate using an agent scorer.
+
+        Args:
+            evaluation_id: The ID of the evaluation being processed
+            evaluation_name: The name of the evaluation
+            input_data: The input data for the evaluation
+            expected_output: The expected output
+            actual_output: The actual output from the agent
+
+        Returns:
+            EvaluationResult containing the score and details
+        """
+        raise NotImplementedError()

uipath/_cli/_evals/_evaluators/_deterministic_evaluator.py

@@ -0,0 +1,41 @@
+from typing import Any, Dict
+
+from .._models import EvaluationResult
+from ._evaluator_base import EvaluatorBase
+
+
+class DeterministicEvaluator(EvaluatorBase):
+    """Evaluator for deterministic/rule-based evaluations."""
+
+    def __init__(self, rule_config: Dict[str, Any], target_output_key: str = "*"):
+        """Initialize the deterministic evaluator.
+
+        Args:
+            rule_config: Configuration for the rule (expected_value, regex_pattern, etc.)
+            target_output_key: Key in output to evaluate ("*" for entire output)
+        """
+        super().__init__()
+        self.rule_config = rule_config or {}
+        self.target_output_key = target_output_key
+
+    async def evaluate(
+        self,
+        evaluation_id: str,
+        evaluation_name: str,
+        input_data: Dict[str, Any],
+        expected_output: Dict[str, Any],
+        actual_output: Dict[str, Any],
+    ) -> EvaluationResult:
+        """Evaluate using deterministic rules.
+
+        Args:
+            evaluation_id: The ID of the evaluation being processed
+            evaluation_name: The name of the evaluation
+            input_data: The input data for the evaluation
+            expected_output: The expected output
+            actual_output: The actual output from the agent
+
+        Returns:
+            EvaluationResult containing the score and details
+        """
+        raise NotImplementedError()

uipath/_cli/_evals/_evaluators/_evaluator_base.py

@@ -0,0 +1,124 @@
+import functools
+import time
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Dict
+
+from uipath._cli._evals._models import (
+    EvaluationResult,
+    EvaluatorCategory,
+    EvaluatorType,
+)
+
+
+def measure_execution_time(func):
+    """Decorator to measure execution time and update EvaluationResult.evaluation_time."""
+
+    @functools.wraps(func)
+    async def wrapper(*args, **kwargs) -> EvaluationResult:
+        start_time = time.time()
+        result = await func(*args, **kwargs)
+        end_time = time.time()
+        execution_time = end_time - start_time
+
+        result.evaluation_time = execution_time
+        return result
+
+    return wrapper
+
+
+@dataclass
+class EvaluatorBaseParams:
+    """Parameters for initializing the base evaluator."""
+
+    evaluator_id: str
+    category: EvaluatorCategory
+    evaluator_type: EvaluatorType
+    name: str
+    description: str
+    created_at: str
+    updated_at: str
+    target_output_key: str
+
+
+class EvaluatorBase(ABC):
+    """Abstract base class for all evaluators."""
+
+    def __init__(self):
+        # initialization done via 'from_params' function
+        self.id: str
+        self.name: str
+        self.description: str
+        self.created_at: str
+        self.updated_at: str
+        self.category: EvaluatorCategory
+        self.type: EvaluatorType
+        self.target_output_key: str
+        pass
+
+    @classmethod
+    def from_params(cls, params: EvaluatorBaseParams, **kwargs):
+        """Initialize the base evaluator from parameters.
+
+        Args:
+            params: EvaluatorBaseParams containing base configuration
+            **kwargs: Additional specific parameters for concrete evaluators
+
+        Returns:
+            Initialized evaluator instance
+        """
+        instance = cls(**kwargs)
+        instance.id = params.evaluator_id
+        instance.category = params.category
+        instance.type = params.evaluator_type
+        instance.name = params.name
+        instance.description = params.description
+        instance.created_at = params.created_at
+        instance.updated_at = params.updated_at
+        instance.target_output_key = params.target_output_key
+        return instance
+
+    @measure_execution_time
+    @abstractmethod
+    async def evaluate(
+        self,
+        evaluation_id: str,
+        evaluation_name: str,
+        input_data: Dict[str, Any],
+        expected_output: Dict[str, Any],
+        actual_output: Dict[str, Any],
+    ) -> EvaluationResult:
+        """Evaluate the given data and return a result.
+
+        Args:
+            evaluation_id: The ID of the evaluation being processed
+            evaluation_name: The name of the evaluation
+            input_data: The input data for the evaluation
+            expected_output: The expected output
+            actual_output: The actual output from the agent
+
+        Returns:
+            EvaluationResult containing the score and details
+        """
+        pass
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the evaluator instance to a dictionary representation.
+
+        Returns:
+            Dict[str, Any]: Dictionary containing all evaluator properties
+        """
+        return {
+            "id": self.id,
+            "name": self.name,
+            "description": self.description,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "category": self.category.name if self.category else None,
+            "type": self.type.name if self.type else None,
+            "target_output_key": self.target_output_key,
+        }
+
+    def __repr__(self) -> str:
+        """String representation of the evaluator."""
+        return f"{self.__class__.__name__}(id='{self.id}', name='{self.name}', category={self.category.name})"

uipath/_cli/_evals/_evaluators/_evaluator_factory.py

@@ -0,0 +1,103 @@
+from typing import Any, Dict
+
+from .._models import EvaluatorCategory, EvaluatorType
+from ._agent_scorer_evaluator import AgentScorerEvaluator
+from ._deterministic_evaluator import DeterministicEvaluator
+from ._evaluator_base import EvaluatorBase, EvaluatorBaseParams
+from ._llm_as_judge_evaluator import LlmAsAJudgeEvaluator
+from ._trajectory_evaluator import TrajectoryEvaluator
+
+
+class EvaluatorFactory:
+    """Factory class for creating evaluator instances based on configuration."""
+
+    @staticmethod
+    def create_evaluator(data: Dict[str, Any]) -> EvaluatorBase:
+        """Create an evaluator instance from configuration data.
+
+        Args:
+            data: Dictionary containing evaluator configuration from JSON file
+
+        Returns:
+            Appropriate evaluator instance based on category
+
+        Raises:
+            ValueError: If category is unknown or required fields are missing
+        """
+        # Extract common fields
+        evaluator_id = data.get("id")
+        if not evaluator_id:
+            raise ValueError("Evaluator configuration must include 'id' field")
+
+        category = EvaluatorCategory.from_int(data.get("category"))
+        evaluator_type = EvaluatorType.from_int(data.get("type", EvaluatorType.Unknown))
+        name = data.get("name", "")
+        description = data.get("description", "")
+        created_at = data.get("createdAt", "")
+        updated_at = data.get("updatedAt", "")
+        target_output_key = data.get("targetOutputKey", "")
+
+        # Create base parameters
+        base_params = EvaluatorBaseParams(
+            evaluator_id=evaluator_id,
+            category=category,
+            evaluator_type=evaluator_type,
+            name=name,
+            description=description,
+            created_at=created_at,
+            updated_at=updated_at,
+            target_output_key=target_output_key,
+        )
+
+        # Create evaluator based on category
+        if category == EvaluatorCategory.Deterministic:
+            return EvaluatorFactory._create_deterministic_evaluator(base_params, data)
+        elif category == EvaluatorCategory.LlmAsAJudge:
+            return EvaluatorFactory._create_llm_as_judge_evaluator(base_params, data)
+        elif category == EvaluatorCategory.AgentScorer:
+            return EvaluatorFactory._create_agent_scorer_evaluator(base_params, data)
+        elif category == EvaluatorCategory.Trajectory:
+            return EvaluatorFactory._create_trajectory_evaluator(base_params, data)
+        else:
+            raise ValueError(f"Unknown evaluator category: {category}")
+
+    @staticmethod
+    def _create_deterministic_evaluator(
+        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+    ) -> DeterministicEvaluator:
+        """Create a deterministic evaluator."""
+        raise NotImplementedError()
+
+    @staticmethod
+    def _create_llm_as_judge_evaluator(
+        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+    ) -> LlmAsAJudgeEvaluator:
+        """Create an LLM-as-a-judge evaluator."""
+        prompt = data.get("prompt", "")
+        if not prompt:
+            raise ValueError("LLM evaluator must include 'prompt' field")
+
+        model = data.get("model", "")
+        if not model:
+            raise ValueError("LLM evaluator must include 'model' field")
+
+        return LlmAsAJudgeEvaluator.from_params(
+            base_params,
+            prompt=prompt,
+            model=model,
+            target_output_key=data.get("targetOutputKey", "*"),
+        )
+
+    @staticmethod
+    def _create_agent_scorer_evaluator(
+        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+    ) -> AgentScorerEvaluator:
+        """Create an agent scorer evaluator."""
+        raise NotImplementedError()
+
+    @staticmethod
+    def _create_trajectory_evaluator(
+        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+    ) -> TrajectoryEvaluator:
+        """Create a trajectory evaluator."""
+        raise NotImplementedError()

uipath/_cli/_evals/_evaluators/_llm_as_judge_evaluator.py

@@ -0,0 +1,181 @@
+import json
+from typing import Any, Dict
+
+from ...._config import Config
+from ...._execution_context import ExecutionContext
+from ...._services.llm_gateway_service import UiPathLlmChatService
+from ...._utils.constants import (
+    ENV_BASE_URL,
+    ENV_UIPATH_ACCESS_TOKEN,
+    ENV_UNATTENDED_USER_ACCESS_TOKEN,
+    COMMUNITY_agents_SUFFIX,
+)
+from .._models import EvaluationResult, LLMResponse
+from ._evaluator_base import EvaluatorBase
+
+
+class LlmAsAJudgeEvaluator(EvaluatorBase):
+    """Evaluator that uses an LLM to judge the quality of outputs."""
+
+    def __init__(self, prompt: str = "", model: str = "", target_output_key: str = "*"):
+        """Initialize the LLM-as-a-judge evaluator.
+
+        Args:
+            prompt: The prompt template for the LLM
+            model: The model to use for evaluation
+            target_output_key: Key in output to evaluate ("*" for entire output)
+        """
+        super().__init__()
+        self.actual_output_placeholder = "{{ActualOutput}}"
+        self.expected_output_placeholder = "{{ExpectedOutput}}"
+        self._initialize_llm()
+        self.prompt = prompt
+        self.model = model
+        self.target_output_key: str = target_output_key
+
+    def _initialize_llm(self):
+        """Initialize the LLM used for evaluation."""
+        import os
+
+        base_url_value: str = os.getenv(ENV_BASE_URL)  # type: ignore
+        secret_value: str = os.getenv(ENV_UNATTENDED_USER_ACCESS_TOKEN) or os.getenv(
+            ENV_UIPATH_ACCESS_TOKEN
+        )  # type: ignore
+        config = Config(
+            base_url=base_url_value,
+            secret=secret_value,
+        )
+        self.llm = UiPathLlmChatService(config, ExecutionContext())
+
+    async def evaluate(
+        self,
+        evaluation_id: str,
+        evaluation_name: str,
+        input_data: Dict[str, Any],
+        expected_output: Dict[str, Any],
+        actual_output: Dict[str, Any],
+    ) -> EvaluationResult:
+        """Evaluate using an LLM as a judge.
+
+        Args:
+            evaluation_id: The ID of the evaluation being processed
+            evaluation_name: The name of the evaluation
+            input_data: The input data for the evaluation
+            expected_output: The expected output
+            actual_output: The actual output from the agent
+
+        Returns:
+            EvaluationResult containing the score and details
+        """
+        # Extract the target value to evaluate
+        target_value = self._extract_target_value(actual_output)
+        expected_value = self._extract_target_value(expected_output)
+
+        # Create the evaluation prompt
+        evaluation_prompt = self._create_evaluation_prompt(expected_value, target_value)
+
+        llm_response = await self._get_llm_response(evaluation_prompt)
+
+        return EvaluationResult(
+            evaluation_id=evaluation_id,
+            evaluation_name=evaluation_name,
+            evaluator_id=self.id,
+            evaluator_name=self.name,
+            score=llm_response.score,
+            input=input_data,
+            expected_output=expected_output,
+            actual_output=actual_output,
+            details=llm_response.justification,
+        )
+
+    def _extract_target_value(self, output: Dict[str, Any]) -> Any:
+        """Extract the target value from output based on target_output_key."""
+        if self.target_output_key == "*":
+            return output
+
+        # Handle nested keys
+        keys = self.target_output_key.split(".")
+        value = output
+
+        try:
+            for key in keys:
+                if isinstance(value, dict):
+                    value = value[key]
+                else:
+                    return None
+            return value
+        except (KeyError, TypeError):
+            return None
+
+    def _create_evaluation_prompt(
+        self, expected_output: Any, actual_output: Any
+    ) -> str:
+        """Create the evaluation prompt for the LLM."""
+        formatted_prompt = self.prompt.replace(
+            self.actual_output_placeholder,
+            str(actual_output),
+        )
+        formatted_prompt = formatted_prompt.replace(
+            self.expected_output_placeholder,
+            str(expected_output),
+        )
+
+        return formatted_prompt
+
+    async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse:
+        """Get response from the LLM.
+
+        Args:
+            evaluation_prompt: The formatted prompt to send to the LLM
+
+        Returns:
+            LLMResponse with score and justification
+        """
+        try:
+            # remove community-agents suffix from llm model name
+            model = self.model
+            if model.endswith(COMMUNITY_agents_SUFFIX):
+                model = model.replace(COMMUNITY_agents_SUFFIX, "")
+
+            # Prepare the request
+            request_data = {
+                "model": model,
+                "messages": [{"role": "user", "content": evaluation_prompt}],
+                "response_format": {
+                    "type": "json_schema",
+                    "json_schema": {
+                        "name": "evaluation_response",
+                        "schema": {
+                            "type": "object",
+                            "properties": {
+                                "score": {
+                                    "type": "number",
+                                    "minimum": 0,
+                                    "maximum": 100,
+                                    "description": "Score between 0 and 100",
+                                },
+                                "justification": {
+                                    "type": "string",
+                                    "description": "Explanation for the score",
+                                },
+                            },
+                            "required": ["score", "justification"],
+                        },
+                    },
+                },
+            }
+
+            response = await self.llm.chat_completions(**request_data)
+
+            try:
+                return LLMResponse(**json.loads(response.choices[-1].message.content))
+            except (json.JSONDecodeError, ValueError) as e:
+                return LLMResponse(
+                    score=0.0, justification=f"Error parsing LLM response: {str(e)}"
+                )
+
+        except Exception as e:
+            # Fallback in case of any errors
+            return LLMResponse(
+                score=0.0, justification=f"Error during LLM evaluation: {str(e)}"
+            )

uipath/_cli/_evals/_evaluators/_trajectory_evaluator.py

@@ -0,0 +1,48 @@
+from typing import Any, Dict
+
+from .._models import EvaluationResult
+from ._evaluator_base import EvaluatorBase
+
+
+class TrajectoryEvaluator(EvaluatorBase):
+    """Evaluator that analyzes the trajectory/path taken to reach outputs."""
+
+    def __init__(
+        self,
+        trajectory_config: Dict[str, Any],
+        step_weights: Dict[str, float],
+        target_output_key: str = "*",
+    ):
+        """Initialize the trajectory evaluator.
+
+        Args:
+            trajectory_config: Configuration for trajectory analysis
+            step_weights: Weights for different steps in the trajectory
+            target_output_key: Key in output to evaluate ("*" for entire output)
+        """
+        super().__init__()
+        self.trajectory_config = trajectory_config or {}
+        self.step_weights = step_weights or {}
+        self.target_output_key = target_output_key
+
+    async def evaluate(
+        self,
+        evaluation_id: str,
+        evaluation_name: str,
+        input_data: Dict[str, Any],
+        expected_output: Dict[str, Any],
+        actual_output: Dict[str, Any],
+    ) -> EvaluationResult:
+        """Evaluate using trajectory analysis.
+
+        Args:
+            evaluation_id: The ID of the evaluation being processed
+            evaluation_name: The name of the evaluation
+            input_data: The input data for the evaluation
+            expected_output: The expected output
+            actual_output: The actual output from the agent
+
+        Returns:
+            EvaluationResult containing the score and details
+        """
+        raise NotImplementedError()

uipath/_cli/_evals/_models/__init__.py

@@ -0,0 +1,18 @@
+from uipath._cli._evals._models._evaluation_set import EvaluationItem, EvaluationSet
+from uipath._cli._evals._models._evaluators import (
+    EvaluationResult,
+    EvaluationSetResult,
+    EvaluatorCategory,
+    EvaluatorType,
+    LLMResponse,
+)
+
+__all__ = [
+    "LLMResponse",
+    "EvaluatorCategory",
+    "EvaluatorType",
+    "EvaluationResult",
+    "EvaluationSetResult",
+    "EvaluationItem",
+    "EvaluationSet",
+]

uipath/_cli/_evals/_models/_evaluation_set.py

@@ -0,0 +1,43 @@
+from enum import IntEnum
+from typing import Any, Dict, List
+
+from pydantic import BaseModel, Field
+
+
+class EvaluationItem(BaseModel):
+    """Individual evaluation item within an evaluation set."""
+
+    id: str
+    name: str
+    inputs: Dict[str, Any]
+    expectedOutput: Dict[str, Any]
+    expectedAgentBehavior: str = ""
+    simulationInstructions: str = ""
+    simulateInput: bool = False
+    inputGenerationInstructions: str = ""
+    simulateTools: bool = False
+    toolsToSimulate: List[str] = Field(default_factory=list)
+    evalSetId: str
+    createdAt: str
+    updatedAt: str
+
+
+class EvaluationSet(BaseModel):
+    """Complete evaluation set model."""
+
+    id: str
+    fileName: str
+    evaluatorRefs: List[str] = Field(default_factory=list)
+    evaluations: List[EvaluationItem] = Field(default_factory=list)
+    name: str
+    batchSize: int = 10
+    timeoutMinutes: int = 20
+    modelSettings: List[Dict[str, Any]] = Field(default_factory=list)
+    createdAt: str
+    updatedAt: str
+
+
+class EvaluationStatus(IntEnum):
+    PENDING = 0
+    IN_PROGRESS = 1
+    COMPLETED = 2