uipath 2.1.52__py3-none-any.whl → 2.1.53__py3-none-any.whl

uipath/_cli/_evals/_evaluators/__init__.py

@@ -1,22 +0,0 @@
-"""Evaluators package for the evaluation system.
-
-This package contains all evaluator types and the factory for creating them.
-"""
-
-from ._deterministic_evaluator_base import DeterministicEvaluatorBase
-from ._evaluator_base import EvaluatorBase
-from ._evaluator_factory import EvaluatorFactory
-from ._exact_match_evaluator import ExactMatchEvaluator
-from ._json_similarity_evaluator import JsonSimilarityEvaluator
-from ._llm_as_judge_evaluator import LlmAsAJudgeEvaluator
-from ._trajectory_evaluator import TrajectoryEvaluator
-
-__all__ = [
-    "EvaluatorBase",
-    "DeterministicEvaluatorBase",
-    "EvaluatorFactory",
-    "JsonSimilarityEvaluator",
-    "ExactMatchEvaluator",
-    "LlmAsAJudgeEvaluator",
-    "TrajectoryEvaluator",
-]

uipath/_cli/_evals/_evaluators/_deterministic_evaluator_base.py

@@ -1,46 +0,0 @@
-import copy
-import json
-from abc import ABC
-from typing import Any, Dict, Tuple
-
-from ._evaluator_base import EvaluatorBase
-
-
-class DeterministicEvaluatorBase(EvaluatorBase, ABC):
-    def __init__(self, target_output_key: str = "*"):
-        super().__init__()
-        self.target_output_key = target_output_key
-
-    def _select_targets(
-        self, expected_output: Dict[str, Any], actual_output: Dict[str, Any]
-    ) -> Tuple[Any, Any]:
-        actual_output_copy = copy.deepcopy(actual_output)
-        expected_output_copy = copy.deepcopy(expected_output)
-        if self.target_output_key != "*":
-            if (
-                self.target_output_key not in actual_output
-                or self.target_output_key not in expected_output
-            ):
-                raise ValueError(
-                    f"Field '{self.target_output_key}' missing from expected or actual output"
-                )
-            actual_output_copy = actual_output_copy[self.target_output_key]
-            expected_output_copy = expected_output[self.target_output_key]
-        return actual_output_copy, expected_output_copy
-
-    def _canonical_json(self, obj: Any) -> str:
-        return json.dumps(
-            self._normalize_numbers(obj),
-            sort_keys=True,
-            separators=(",", ":"),
-            ensure_ascii=False,
-        )
-
-    def _normalize_numbers(self, obj: Any) -> Any:
-        if isinstance(obj, dict):
-            return {k: self._normalize_numbers(v) for k, v in obj.items()}
-        if isinstance(obj, (list, tuple)):
-            return [self._normalize_numbers(v) for v in obj]
-        if isinstance(obj, (int, float)) and not isinstance(obj, bool):
-            return float(obj)
-        return obj

uipath/_cli/_evals/_evaluators/_evaluator_base.py

@@ -1,124 +0,0 @@
-import functools
-import time
-from abc import ABC, abstractmethod
-from dataclasses import dataclass
-from typing import Any, Dict
-
-from uipath._cli._evals._models import (
-    EvaluationResult,
-    EvaluatorCategory,
-    EvaluatorType,
-)
-
-
-def measure_execution_time(func):
-    """Decorator to measure execution time and update EvaluationResult.evaluation_time."""
-
-    @functools.wraps(func)
-    async def wrapper(*args, **kwargs) -> EvaluationResult:
-        start_time = time.time()
-        result = await func(*args, **kwargs)
-        end_time = time.time()
-        execution_time = end_time - start_time
-
-        result.evaluation_time = execution_time
-        return result
-
-    return wrapper
-
-
-@dataclass
-class EvaluatorBaseParams:
-    """Parameters for initializing the base evaluator."""
-
-    evaluator_id: str
-    category: EvaluatorCategory
-    evaluator_type: EvaluatorType
-    name: str
-    description: str
-    created_at: str
-    updated_at: str
-    target_output_key: str
-
-
-class EvaluatorBase(ABC):
-    """Abstract base class for all evaluators."""
-
-    def __init__(self):
-        # initialization done via 'from_params' function
-        self.id: str
-        self.name: str
-        self.description: str
-        self.created_at: str
-        self.updated_at: str
-        self.category: EvaluatorCategory
-        self.type: EvaluatorType
-        self.target_output_key: str
-        pass
-
-    @classmethod
-    def from_params(cls, params: EvaluatorBaseParams, **kwargs):
-        """Initialize the base evaluator from parameters.
-
-        Args:
-            params: EvaluatorBaseParams containing base configuration
-            **kwargs: Additional specific parameters for concrete evaluators
-
-        Returns:
-            Initialized evaluator instance
-        """
-        instance = cls(**kwargs)
-        instance.id = params.evaluator_id
-        instance.category = params.category
-        instance.type = params.evaluator_type
-        instance.name = params.name
-        instance.description = params.description
-        instance.created_at = params.created_at
-        instance.updated_at = params.updated_at
-        instance.target_output_key = params.target_output_key
-        return instance
-
-    @measure_execution_time
-    @abstractmethod
-    async def evaluate(
-        self,
-        evaluation_id: str,
-        evaluation_name: str,
-        input_data: Dict[str, Any],
-        expected_output: Dict[str, Any],
-        actual_output: Dict[str, Any],
-    ) -> EvaluationResult:
-        """Evaluate the given data and return a result.
-
-        Args:
-            evaluation_id: The ID of the evaluation being processed
-            evaluation_name: The name of the evaluation
-            input_data: The input data for the evaluation
-            expected_output: The expected output
-            actual_output: The actual output from the agent
-
-        Returns:
-            EvaluationResult containing the score and details
-        """
-        pass
-
-    def to_dict(self) -> Dict[str, Any]:
-        """Convert the evaluator instance to a dictionary representation.
-
-        Returns:
-            Dict[str, Any]: Dictionary containing all evaluator properties
-        """
-        return {
-            "id": self.id,
-            "name": self.name,
-            "description": self.description,
-            "created_at": self.created_at,
-            "updated_at": self.updated_at,
-            "category": self.category.name if self.category else None,
-            "type": self.type.name if self.type else None,
-            "target_output_key": self.target_output_key,
-        }
-
-    def __repr__(self) -> str:
-        """String representation of the evaluator."""
-        return f"{self.__class__.__name__}(id='{self.id}', name='{self.name}', category={self.category.name})"

uipath/_cli/_evals/_evaluators/_exact_match_evaluator.py

@@ -1,40 +0,0 @@
-import copy
-from typing import Any, Dict
-
-from uipath._cli._evals._evaluators._deterministic_evaluator_base import (
-    DeterministicEvaluatorBase,
-)
-from uipath._cli._evals._models import EvaluationResult
-from uipath._cli._evals._models._evaluators import ScoreType
-
-
-class ExactMatchEvaluator(DeterministicEvaluatorBase):
-    async def evaluate(
-        self,
-        evaluation_id: str,
-        evaluation_name: str,
-        input_data: Dict[str, Any],
-        expected_output: Dict[str, Any],
-        actual_output: Dict[str, Any],
-    ) -> EvaluationResult:
-        actual_output_copy = copy.deepcopy(actual_output)
-        expected_output_copy = copy.deepcopy(expected_output)
-
-        actual_output, expected_output = self._select_targets(
-            expected_output, actual_output
-        )
-        are_equal = self._canonical_json(actual_output) == self._canonical_json(
-            expected_output
-        )
-
-        return EvaluationResult(
-            evaluation_id=evaluation_id,
-            evaluation_name=evaluation_name,
-            evaluator_id=self.id,
-            evaluator_name=self.name,
-            score=are_equal,
-            input=input_data,
-            expected_output=expected_output_copy,
-            actual_output=actual_output_copy,
-            score_type=ScoreType.BOOLEAN,
-        )

uipath/_cli/_evals/_evaluators/_llm_as_judge_evaluator.py

@@ -1,183 +0,0 @@
-import json
-from typing import Any, Dict
-
-from ...._config import Config
-from ...._execution_context import ExecutionContext
-from ...._services.llm_gateway_service import UiPathLlmChatService
-from ...._utils.constants import (
-    ENV_BASE_URL,
-    ENV_UIPATH_ACCESS_TOKEN,
-    ENV_UNATTENDED_USER_ACCESS_TOKEN,
-    COMMUNITY_agents_SUFFIX,
-)
-from .._models import EvaluationResult, LLMResponse
-from .._models._evaluators import ScoreType
-from ._evaluator_base import EvaluatorBase
-
-
-class LlmAsAJudgeEvaluator(EvaluatorBase):
-    """Evaluator that uses an LLM to judge the quality of outputs."""
-
-    def __init__(self, prompt: str = "", model: str = "", target_output_key: str = "*"):
-        """Initialize the LLM-as-a-judge evaluator.
-
-        Args:
-            prompt: The prompt template for the LLM
-            model: The model to use for evaluation
-            target_output_key: Key in output to evaluate ("*" for entire output)
-        """
-        super().__init__()
-        self.actual_output_placeholder = "{{ActualOutput}}"
-        self.expected_output_placeholder = "{{ExpectedOutput}}"
-        self._initialize_llm()
-        self.prompt = prompt
-        self.model = model
-        self.target_output_key: str = target_output_key
-
-    def _initialize_llm(self):
-        """Initialize the LLM used for evaluation."""
-        import os
-
-        base_url_value: str = os.getenv(ENV_BASE_URL)  # type: ignore
-        secret_value: str = os.getenv(ENV_UNATTENDED_USER_ACCESS_TOKEN) or os.getenv(
-            ENV_UIPATH_ACCESS_TOKEN
-        )  # type: ignore
-        config = Config(
-            base_url=base_url_value,
-            secret=secret_value,
-        )
-        self.llm = UiPathLlmChatService(config, ExecutionContext())
-
-    async def evaluate(
-        self,
-        evaluation_id: str,
-        evaluation_name: str,
-        input_data: Dict[str, Any],
-        expected_output: Dict[str, Any],
-        actual_output: Dict[str, Any],
-    ) -> EvaluationResult:
-        """Evaluate using an LLM as a judge.
-
-        Args:
-            evaluation_id: The ID of the evaluation being processed
-            evaluation_name: The name of the evaluation
-            input_data: The input data for the evaluation
-            expected_output: The expected output
-            actual_output: The actual output from the agent
-
-        Returns:
-            EvaluationResult containing the score and details
-        """
-        # Extract the target value to evaluate
-        target_value = self._extract_target_value(actual_output)
-        expected_value = self._extract_target_value(expected_output)
-
-        # Create the evaluation prompt
-        evaluation_prompt = self._create_evaluation_prompt(expected_value, target_value)
-
-        llm_response = await self._get_llm_response(evaluation_prompt)
-
-        return EvaluationResult(
-            evaluation_id=evaluation_id,
-            evaluation_name=evaluation_name,
-            evaluator_id=self.id,
-            evaluator_name=self.name,
-            score=llm_response.score,
-            input=input_data,
-            expected_output=expected_output,
-            actual_output=actual_output,
-            details=llm_response.justification,
-            score_type=ScoreType.NUMERICAL,
-        )
-
-    def _extract_target_value(self, output: Dict[str, Any]) -> Any:
-        """Extract the target value from output based on target_output_key."""
-        if self.target_output_key == "*":
-            return output
-
-        # Handle nested keys
-        keys = self.target_output_key.split(".")
-        value = output
-
-        try:
-            for key in keys:
-                if isinstance(value, dict):
-                    value = value[key]
-                else:
-                    return None
-            return value
-        except (KeyError, TypeError):
-            return None
-
-    def _create_evaluation_prompt(
-        self, expected_output: Any, actual_output: Any
-    ) -> str:
-        """Create the evaluation prompt for the LLM."""
-        formatted_prompt = self.prompt.replace(
-            self.actual_output_placeholder,
-            str(actual_output),
-        )
-        formatted_prompt = formatted_prompt.replace(
-            self.expected_output_placeholder,
-            str(expected_output),
-        )
-
-        return formatted_prompt
-
-    async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse:
-        """Get response from the LLM.
-
-        Args:
-            evaluation_prompt: The formatted prompt to send to the LLM
-
-        Returns:
-            LLMResponse with score and justification
-        """
-        try:
-            # remove community-agents suffix from llm model name
-            model = self.model
-            if model.endswith(COMMUNITY_agents_SUFFIX):
-                model = model.replace(COMMUNITY_agents_SUFFIX, "")
-
-            # Prepare the request
-            request_data = {
-                "model": model,
-                "messages": [{"role": "user", "content": evaluation_prompt}],
-                "response_format": {
-                    "type": "json_schema",
-                    "json_schema": {
-                        "name": "evaluation_response",
-                        "schema": {
-                            "type": "object",
-                            "properties": {
-                                "score": {
-                                    "type": "number",
-                                    "minimum": 0,
-                                    "maximum": 100,
-                                    "description": "Score between 0 and 100",
-                                },
-                                "justification": {
-                                    "type": "string",
-                                    "description": "Explanation for the score",
-                                },
-                            },
-                            "required": ["score", "justification"],
-                        },
-                    },
-                },
-            }
-
-            response = await self.llm.chat_completions(**request_data)
-
-            try:
-                return LLMResponse(**json.loads(response.choices[-1].message.content))
-            except (json.JSONDecodeError, ValueError) as e:
-                return LLMResponse(
-                    score=0.0, justification=f"Error parsing LLM response: {str(e)}"
-                )
-
-        except Exception as e:
-            # Fallback in case of any errors
-            return LLMResponse(
-                score=0.0, justification=f"Error during LLM evaluation: {str(e)}"
-            )

uipath/_cli/_evals/_evaluators/_trajectory_evaluator.py

@@ -1,48 +0,0 @@
-from typing import Any, Dict
-
-from .._models import EvaluationResult
-from ._evaluator_base import EvaluatorBase
-
-
-class TrajectoryEvaluator(EvaluatorBase):
-    """Evaluator that analyzes the trajectory/path taken to reach outputs."""
-
-    def __init__(
-        self,
-        trajectory_config: Dict[str, Any],
-        step_weights: Dict[str, float],
-        target_output_key: str = "*",
-    ):
-        """Initialize the trajectory evaluator.
-
-        Args:
-            trajectory_config: Configuration for trajectory analysis
-            step_weights: Weights for different steps in the trajectory
-            target_output_key: Key in output to evaluate ("*" for entire output)
-        """
-        super().__init__()
-        self.trajectory_config = trajectory_config or {}
-        self.step_weights = step_weights or {}
-        self.target_output_key = target_output_key
-
-    async def evaluate(
-        self,
-        evaluation_id: str,
-        evaluation_name: str,
-        input_data: Dict[str, Any],
-        expected_output: Dict[str, Any],
-        actual_output: Dict[str, Any],
-    ) -> EvaluationResult:
-        """Evaluate using trajectory analysis.
-
-        Args:
-            evaluation_id: The ID of the evaluation being processed
-            evaluation_name: The name of the evaluation
-            input_data: The input data for the evaluation
-            expected_output: The expected output
-            actual_output: The actual output from the agent
-
-        Returns:
-            EvaluationResult containing the score and details
-        """
-        raise NotImplementedError()

uipath/_cli/_evals/_models/__init__.py

@@ -1,18 +0,0 @@
-from uipath._cli._evals._models._evaluation_set import EvaluationItem, EvaluationSet
-from uipath._cli._evals._models._evaluators import (
-    EvaluationResult,
-    EvaluationSetResult,
-    EvaluatorCategory,
-    EvaluatorType,
-    LLMResponse,
-)
-
-__all__ = [
-    "LLMResponse",
-    "EvaluatorCategory",
-    "EvaluatorType",
-    "EvaluationResult",
-    "EvaluationSetResult",
-    "EvaluationItem",
-    "EvaluationSet",
-]

uipath/_cli/_evals/_models/_agent_execution_output.py

@@ -1,14 +0,0 @@
-from opentelemetry.sdk.trace import ReadableSpan
-from pydantic import BaseModel, ConfigDict
-
-from uipath._cli._runtime._contracts import UiPathRuntimeResult
-
-
-class UiPathEvalRunExecutionOutput(BaseModel):
-    """Result of a single agent response."""
-
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-
-    execution_time: float
-    spans: list[ReadableSpan]
-    result: UiPathRuntimeResult