uipath 2.1.71__py3-none-any.whl → 2.1.73__py3-none-any.whl

uipath/_cli/_dev/_terminal/_components/_history.py

@@ -73,13 +73,33 @@ class RunHistoryPanel(Container):
         self.refresh_list()
 
     def _refresh_running_items(self) -> None:
+        """Refresh display names for running items only."""
         if not any(run.status == "running" for run in self.runs):
-            return None  # No running items, skip update
+            return None
 
-        run_list = self.query_one("#run-list", ListView)
+        try:
+            run_list = self.query_one("#run-list", ListView)
+        except Exception:
+            return None
+
+        # Take a snapshot of items to avoid mid-iteration changes
+        items_snapshot = list(run_list.children)
+
+        for item in items_snapshot:
+            if not hasattr(item, "run_id"):
+                continue
+
+            run = self.get_run_by_id(item.run_id)
+            if not run or run.status != "running":
+                continue
+
+            # Check if item still exists in the list (wasn't removed)
+            if item not in run_list.children:
+                continue
 
-        for item in run_list.children:
-            run = self.get_run_by_id(item.run_id)  # type: ignore[attr-defined]
-            if run and run.status == "running":
+            try:
                 static = item.query_one(Static)
                 static.update(run.display_name)
+            except Exception:
+                # Item structure changed or was removed
+                continue

uipath/_cli/_evals/_evaluator_factory.py

@@ -1,5 +1,14 @@
 from typing import Any, Dict
 
+from pydantic import TypeAdapter
+
+from uipath._cli._evals._models._evaluator import (
+    EqualsEvaluatorParams,
+    Evaluator,
+    JsonSimilarityEvaluatorParams,
+    LLMEvaluatorParams,
+    TrajectoryEvaluatorParams,
+)
 from uipath._cli._evals._models._evaluator_base_params import EvaluatorBaseParams
 from uipath.eval.evaluators import (
     BaseEvaluator,
@@ -8,7 +17,6 @@ from uipath.eval.evaluators import (
     LlmAsAJudgeEvaluator,
     TrajectoryEvaluator,
 )
-from uipath.eval.models.models import EvaluatorCategory, EvaluatorType
 
 
 class EvaluatorFactory:
@@ -35,110 +43,64 @@ class EvaluatorFactory:
         if not id:
             raise ValueError("Evaluator configuration must include 'id' field")
 
-        category = EvaluatorCategory.from_int(data.get("category"))
-        evaluator_type = EvaluatorType.from_int(data.get("type", EvaluatorType.Unknown))
-        description = data.get("description", "")
-        created_at = data.get("createdAt", "")
-        updated_at = data.get("updatedAt", "")
-        target_output_key = data.get("targetOutputKey", "")
-
-        # Create base parameters
-        base_params = EvaluatorBaseParams(
-            id=id,
-            category=category,
-            evaluator_type=evaluator_type,
-            name=name,
-            description=description,
-            created_at=created_at,
-            updated_at=updated_at,
-            target_output_key=target_output_key,
-        )
-
-        match category:
-            case EvaluatorCategory.Deterministic:
-                if evaluator_type == evaluator_type.Equals:
-                    return EvaluatorFactory._create_exact_match_evaluator(
-                        base_params, data
-                    )
-                elif evaluator_type == evaluator_type.JsonSimilarity:
-                    return EvaluatorFactory._create_json_similarity_evaluator(
-                        base_params, data
-                    )
-                else:
-                    raise ValueError(
-                        f"Unknown evaluator type {evaluator_type} for category {category}"
-                    )
-            case EvaluatorCategory.LlmAsAJudge:
-                return EvaluatorFactory._create_llm_as_judge_evaluator(
-                    base_params, data
-                )
-            case EvaluatorCategory.AgentScorer:
-                raise NotImplementedError()
-            case EvaluatorCategory.Trajectory:
-                return EvaluatorFactory._create_trajectory_evaluator(base_params, data)
+        params: EvaluatorBaseParams = TypeAdapter(Evaluator).validate_python(data)
+
+        match params:
+            case EqualsEvaluatorParams():
+                return EvaluatorFactory._create_exact_match_evaluator(params)
+            case JsonSimilarityEvaluatorParams():
+                return EvaluatorFactory._create_json_similarity_evaluator(params)
+            case LLMEvaluatorParams():
+                return EvaluatorFactory._create_llm_as_judge_evaluator(params)
+            case TrajectoryEvaluatorParams():
+                return EvaluatorFactory._create_trajectory_evaluator(params)
             case _:
-                raise ValueError(f"Unknown evaluator category: {category}")
+                raise ValueError(f"Unknown evaluator category: {params}")
 
     @staticmethod
     def _create_exact_match_evaluator(
-        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+        params: EqualsEvaluatorParams,
     ) -> ExactMatchEvaluator:
         """Create a deterministic evaluator."""
-        return ExactMatchEvaluator(
-            **base_params.model_dump(),
-        )
+        return ExactMatchEvaluator(**params.model_dump())
 
     @staticmethod
     def _create_json_similarity_evaluator(
-        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+        params: JsonSimilarityEvaluatorParams,
     ) -> JsonSimilarityEvaluator:
         """Create a deterministic evaluator."""
-        return JsonSimilarityEvaluator(
-            **base_params.model_dump(),
-        )
+        return JsonSimilarityEvaluator(**params.model_dump())
 
     @staticmethod
     def _create_llm_as_judge_evaluator(
-        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+        params: LLMEvaluatorParams,
     ) -> LlmAsAJudgeEvaluator:
         """Create an LLM-as-a-judge evaluator."""
-        prompt = data.get("prompt", "")
-        if not prompt:
+        if not params.prompt:
             raise ValueError("LLM evaluator must include 'prompt' field")
 
-        model = data.get("model", "")
-        if not model:
+        if not params.model:
             raise ValueError("LLM evaluator must include 'model' field")
-        if model == "same-as-agent":
+        if params.model == "same-as-agent":
             raise ValueError(
                 "'same-as-agent' model option is not supported by coded agents evaluations. Please select a specific model for the evaluator."
             )
 
-        return LlmAsAJudgeEvaluator(
-            **base_params.model_dump(),
-            prompt=prompt,
-            model=model,
-        )
+        return LlmAsAJudgeEvaluator(**params.model_dump())
 
     @staticmethod
     def _create_trajectory_evaluator(
-        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+        params: TrajectoryEvaluatorParams,
     ) -> TrajectoryEvaluator:
         """Create a trajectory evaluator."""
-        prompt = data.get("prompt", "")
-        if not prompt:
+        if not params.prompt:
             raise ValueError("Trajectory evaluator must include 'prompt' field")
 
-        model = data.get("model", "")
-        if not model:
+        if not params.model:
             raise ValueError("LLM evaluator must include 'model' field")
-        if model == "same-as-agent":
+        if params.model == "same-as-agent":
             raise ValueError(
                 "'same-as-agent' model option is not supported by coded agents evaluations. Please select a specific model for the evaluator."
             )
 
-        return TrajectoryEvaluator(
-            **base_params.model_dump(),
-            prompt=prompt,
-            model=model,
-        )
+        return TrajectoryEvaluator(**params.model_dump())

uipath/_cli/_evals/_models/_evaluation_set.py

@@ -1,28 +1,135 @@
-from enum import IntEnum
-from typing import Any, Dict, List
+from enum import Enum, IntEnum
+from typing import Annotated, Any, Dict, List, Literal, Optional, Union
 
 from pydantic import BaseModel, ConfigDict, Field
 from pydantic.alias_generators import to_camel
 
 
+class EvaluationSimulationTool(BaseModel):
+    name: str = Field(..., alias="name")
+
+
+class MockingStrategyType(str, Enum):
+    LLM = "llm"
+    MOCKITO = "mockito"
+    UNKNOWN = "unknown"
+
+
+class BaseMockingStrategy(BaseModel):
+    pass
+
+
+class LLMMockingStrategy(BaseMockingStrategy):
+    type: Literal[MockingStrategyType.LLM] = MockingStrategyType.LLM
+    prompt: str = Field(..., alias="prompt")
+    tools_to_simulate: list[EvaluationSimulationTool] = Field(
+        ..., alias="toolsToSimulate"
+    )
+
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+"""
+{
+            "function": "postprocess",
+            "arguments": {
+              "args": [],
+              "kwargs": {"x": 3}
+            },
+            "then": [
+              {
+                "return": 3
+              },
+              {
+                "raise": {
+                  "__target__": "NotImplementedError"
+                }
+              }
+            ]
+          }
+          """
+
+
+class MockingArgument(BaseModel):
+    args: List[Any] = Field(default_factory=lambda: [], alias="args")
+    kwargs: Dict[str, Any] = Field(default_factory=lambda: {}, alias="kwargs")
+
+
+class MockingAnswerType(str, Enum):
+    RETURN = "return"
+    RAISE = "raise"
+
+
+class MockingAnswer(BaseModel):
+    type: MockingAnswerType
+    value: Any = Field(..., alias="value")
+
+
+class MockingBehavior(BaseModel):
+    function: str = Field(..., alias="function")
+    arguments: MockingArgument = Field(..., alias="arguments")
+    then: List[MockingAnswer] = Field(..., alias="then")
+
+
+class MockitoMockingStrategy(BaseMockingStrategy):
+    type: Literal[MockingStrategyType.MOCKITO] = MockingStrategyType.MOCKITO
+    behaviors: List[MockingBehavior] = Field(..., alias="config")
+
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+KnownMockingStrategy = Annotated[
+    Union[LLMMockingStrategy, MockitoMockingStrategy],
+    Field(discriminator="type"),
+]
+
+
+class UnknownMockingStrategy(BaseMockingStrategy):
+    type: str = Field(..., alias="type")
+
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+MockingStrategy = Union[KnownMockingStrategy, UnknownMockingStrategy]
+
+
+def migrate_mocking_strategy(data) -> MockingStrategy:
+    if data.get("simulate_tools") and "tools_to_simulate" in data:
+        return LLMMockingStrategy(
+            **{
+                "prompt": data["simulation_instructions"],
+                "toolsToSimulate": data["tools_to_simulate"],
+            }
+        )
+    else:
+        return UnknownMockingStrategy(type=MockingStrategyType.UNKNOWN)
+
+
 class EvaluationItem(BaseModel):
     """Individual evaluation item within an evaluation set."""
 
-    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+    model_config = ConfigDict(
+        alias_generator=to_camel, populate_by_name=True, extra="allow"
+    )
 
     id: str
     name: str
     inputs: Dict[str, Any]
     expected_output: Dict[str, Any]
-    expected_agent_behavior: str = ""
-    simulation_instructions: str = ""
-    simulate_input: bool = False
-    input_generation_instructions: str = ""
-    simulate_tools: bool = False
-    tools_to_simulate: List[str] = Field(default_factory=list)
-    eval_set_id: str
-    created_at: str
-    updated_at: str
+    expected_agent_behavior: str = Field(default="", alias="expectedAgentBehavior")
+    eval_set_id: str = Field(alias="evalSetId")
+    created_at: str = Field(alias="createdAt")
+    updated_at: str = Field(alias="updatedAt")
+    mocking_strategy: Optional[MockingStrategy] = Field(
+        default=None,
+        alias="mockingStrategy",
+    )
 
 
 class EvaluationSet(BaseModel):
@@ -31,15 +138,17 @@ class EvaluationSet(BaseModel):
     model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
 
     id: str
-    file_name: str
+    file_name: str = Field(..., alias="fileName")
     evaluator_refs: List[str] = Field(default_factory=list)
     evaluations: List[EvaluationItem] = Field(default_factory=list)
     name: str
-    batch_size: int = 10
-    timeout_minutes: int = 20
-    model_settings: List[Dict[str, Any]] = Field(default_factory=list)
-    created_at: str
-    updated_at: str
+    batch_size: int = Field(10, alias="batchSize")
+    timeout_minutes: int = Field(default=20, alias="timeoutMinutes")
+    model_settings: List[Dict[str, Any]] = Field(
+        default_factory=list, alias="modelSettings"
+    )
+    created_at: str = Field(alias="createdAt")
+    updated_at: str = Field(alias="updatedAt")
 
     def extract_selected_evals(self, eval_ids) -> None:
         selected_evals: list[EvaluationItem] = []

uipath/_cli/_evals/_models/_evaluator.py

@@ -0,0 +1,106 @@
+from typing import Annotated, Any, Literal, Union
+
+from pydantic import BaseModel, ConfigDict, Discriminator, Field, Tag
+
+from uipath.eval.models.models import EvaluatorCategory, EvaluatorType
+
+
+class EvaluatorBaseParams(BaseModel):
+    """Parameters for initializing the base evaluator."""
+
+    id: str
+    name: str
+    description: str
+    evaluator_type: EvaluatorType = Field(..., alias="type")
+    created_at: str = Field(..., alias="createdAt")
+    updated_at: str = Field(..., alias="updatedAt")
+    target_output_key: str = Field(..., alias="targetOutputKey")
+    file_name: str = Field(..., alias="fileName")
+
+
+class LLMEvaluatorParams(EvaluatorBaseParams):
+    category: Literal[EvaluatorCategory.LlmAsAJudge] = Field(..., alias="category")
+    prompt: str = Field(..., alias="prompt")
+    model: str = Field(..., alias="model")
+
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+class TrajectoryEvaluatorParams(EvaluatorBaseParams):
+    category: Literal[EvaluatorCategory.Trajectory] = Field(..., alias="category")
+    prompt: str = Field(..., alias="prompt")
+    model: str = Field(..., alias="model")
+
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+class EqualsEvaluatorParams(EvaluatorBaseParams):
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+class JsonSimilarityEvaluatorParams(EvaluatorBaseParams):
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+class UnknownEvaluatorParams(EvaluatorBaseParams):
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+def evaluator_discriminator(data: Any) -> str:
+    if isinstance(data, dict):
+        category = data.get("category")
+        evaluator_type = data.get("type")
+        match category:
+            case EvaluatorCategory.LlmAsAJudge:
+                return "LLMEvaluatorParams"
+            case EvaluatorCategory.Trajectory:
+                return "TrajectoryEvaluatorParams"
+            case EvaluatorCategory.Deterministic:
+                match evaluator_type:
+                    case EvaluatorType.Equals:
+                        return "EqualsEvaluatorParams"
+                    case EvaluatorType.JsonSimilarity:
+                        return "JsonSimilarityEvaluatorParams"
+                    case _:
+                        return "UnknownEvaluatorParams"
+            case _:
+                return "UnknownEvaluatorParams"
+    else:
+        return "UnknownEvaluatorParams"
+
+
+Evaluator = Annotated[
+    Union[
+        Annotated[
+            LLMEvaluatorParams,
+            Tag("LLMEvaluatorParams"),
+        ],
+        Annotated[
+            TrajectoryEvaluatorParams,
+            Tag("TrajectoryEvaluatorParams"),
+        ],
+        Annotated[
+            EqualsEvaluatorParams,
+            Tag("EqualsEvaluatorParams"),
+        ],
+        Annotated[
+            JsonSimilarityEvaluatorParams,
+            Tag("JsonSimilarityEvaluatorParams"),
+        ],
+        Annotated[
+            UnknownEvaluatorParams,
+            Tag("UnknownEvaluatorParams"),
+        ],
+    ],
+    Field(discriminator=Discriminator(evaluator_discriminator)),
+]

uipath/_cli/_evals/_runtime.py

@@ -35,6 +35,7 @@ from ._models._output import (
     UiPathEvalOutput,
     UiPathEvalRunExecutionOutput,
 )
+from .mocks.mocks import set_evaluation_item
 
 T = TypeVar("T", bound=UiPathBaseRuntime)
 C = TypeVar("C", bound=UiPathRuntimeContext)
@@ -137,6 +138,7 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
             evaluation_set_name=evaluation_set.name, score=0, evaluation_set_results=[]
         )
         for eval_item in evaluation_set.evaluations:
+            set_evaluation_item(eval_item)
             await event_bus.publish(
                 EvaluationEvents.CREATE_EVAL_RUN,
                 EvalRunCreatedEvent(

uipath/_cli/_evals/mocks/__init__.py

@@ -0,0 +1 @@
+"""UiPath mocking framework."""

uipath/_cli/_evals/mocks/llm_mocker.py

@@ -0,0 +1,153 @@
+"""LLM mocker implementation."""
+
+import json
+import logging
+from typing import Any, Callable
+
+from pydantic import BaseModel
+
+from uipath._cli._evals._models._evaluation_set import (
+    EvaluationItem,
+    LLMMockingStrategy,
+)
+from uipath._cli._evals.mocks.mocker import Mocker, R, T, UiPathMockingNoMatcherError
+
+PROMPT = """You are simulating a tool call for automated testing purposes of an Agent.
+Your task is to provide a realistic response for the tool based on its schema, examples, and context.
+
+SELECTED TOOL INFORMATION:
+{toolInfo}
+
+SELECTED TOOL SCHEMA:
+{toolSchema}
+
+SELECTED TOOL EXAMPLES:
+{toolRunExamples}
+
+CHOSEN TOOL INPUT:
+{currentToolInput}
+
+CURRENT AGENT RUN SO FAR:
+{testRunHistory}
+
+HERE IS SOME INFORMATION ABOUT THE AGENT: DO NOT USE THIS INFORMATION TO MAKE THE RESPONSE, BUT RATHER TO UNDERSTAND THE CONTEXT IN WHICH THE TOOL IS BEING USED.
+{agentInfo}
+
+TEST RUN PROCTOR INSTRUCTIONS:
+You will need to simulate a real user's interaction with the tool.  This may require following some run specific instructions. If run instructions are provided, follow them exactly.
+Here are the instructions for this run:
+{testRunProctorInstructions}
+
+Based on the above information, provide a realistic response for this tool call. Your response should:
+1. Match the expected output format according to the tool schema
+2. Be very consistent with how the tool has responded in previous examples. Do no omit fields or properties.
+3. Always include the entire output regardless of token length.
+3. Consider the context of the current test run and the agent being tested.  If the agent is acting on a property, make sure the output includes that property.
+
+Respond ONLY with valid JSON that would be a realistic and completetool response. Do not include any explanations or markdown.
+"""
+
+logger = logging.getLogger(__name__)
+
+
+def pydantic_to_dict_safe(obj: Any) -> Any:
+    """Serialize nested pydantic models to a dict."""
+    if isinstance(obj, BaseModel):
+        # Convert Pydantic model to dict recursively
+        return obj.model_dump(mode="json")
+    elif isinstance(obj, dict):
+        # Recursively convert dict entries
+        return {k: pydantic_to_dict_safe(v) for k, v in obj.items()}
+    elif isinstance(obj, list):
+        # Recursively convert items in lists
+        return [pydantic_to_dict_safe(item) for item in obj]
+    return obj  # Return other types as is
+
+
+class LLMMocker(Mocker):
+    """LLM Based Mocker."""
+
+    def __init__(self, evaluation_item: EvaluationItem):
+        """LLM Mocker constructor."""
+        self.evaluation_item = evaluation_item
+        assert isinstance(self.evaluation_item.mocking_strategy, LLMMockingStrategy)
+
+    async def response(
+        self, func: Callable[[T], R], params: dict[str, Any], *args: T, **kwargs
+    ) -> R:
+        """Respond with mocked response generated by an LLM."""
+        assert isinstance(self.evaluation_item.mocking_strategy, LLMMockingStrategy)
+
+        function_name = params.get("name") or func.__name__
+        if function_name in [
+            x.name for x in self.evaluation_item.mocking_strategy.tools_to_simulate
+        ]:
+            from uipath import UiPath
+            from uipath._services.llm_gateway_service import _cleanup_schema
+
+            llm = UiPath().llm
+            return_type: Any = func.__annotations__.get("return", None)
+            if return_type is None:
+                return_type = Any
+
+            class OutputSchema(BaseModel):
+                response: return_type
+
+            response_format = {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": OutputSchema.__name__.lower(),
+                    "strict": True,
+                    "schema": _cleanup_schema(OutputSchema),
+                },
+            }
+            try:
+                prompt_input: dict[str, Any] = {
+                    "toolRunExamples": [],  # Taken from history. Contains id, input json, output json
+                    "testRunHistory": [],  # This should contain ordered spans.
+                    "toolInfo": {
+                        "name": function_name,
+                        "description": params.get("description"),
+                        "arguments": params.get(
+                            "arguments"
+                        ),  # arguments could be passed into tool
+                        "settings": params.get(
+                            "settings"
+                        ),  # settings could be passed into tool
+                        "inputSchema": params.get("input_schema"),
+                    },
+                    "toolSchema": params.get("input_schema"),
+                    "currentToolInput": {
+                        "args": args,
+                        "kwargs": kwargs,
+                    },
+                    "agentInfo": {  # This is incomplete
+                        # "agentName": self.evaluation_item.name, # to be obtained.
+                        "actionName": self.evaluation_item.name,  # Not sure if this is right?
+                        "userInput": self.evaluation_item.inputs,
+                    },
+                    "testRunProctorInstructions": self.evaluation_item.mocking_strategy.prompt,
+                }
+                prompt_input = {
+                    k: json.dumps(pydantic_to_dict_safe(v))
+                    for k, v in prompt_input.items()
+                }
+                response = await llm.chat_completions(
+                    [
+                        {
+                            "role": "user",
+                            "content": PROMPT.format(**prompt_input),
+                        },
+                    ],
+                    response_format=response_format,
+                )
+                mocked_response = OutputSchema(
+                    **json.loads(response.choices[0].message.content)
+                )
+                return mocked_response.response
+            except Exception:
+                raise
+        else:
+            raise UiPathMockingNoMatcherError(
+                f"Method '{function_name}' is not simulated."
+            )

uipath/_cli/_evals/mocks/mocker.py

@@ -0,0 +1,29 @@
+"""Mocker definitions and implementations."""
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from typing import Any, TypeVar
+
+T = TypeVar("T")
+R = TypeVar("R")
+
+
+class Mocker(ABC):
+    """Mocker interface."""
+
+    @abstractmethod
+    async def response(
+        self,
+        func: Callable[[T], R],
+        params: dict[str, Any],
+        *args: T,
+        **kwargs,
+    ) -> R:
+        """Respond with mocked response."""
+        raise NotImplementedError()
+
+
+class UiPathMockingNoMatcherError(Exception):
+    """Exception when a mocker is unable to find a match with the invocation."""
+
+    pass