uipath 2.1.72__tar.gz → 2.1.74__tar.gz

{uipath-2.1.72 → uipath-2.1.74}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: uipath
-Version: 2.1.72
+Version: 2.1.74
 Summary: Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools.
 Project-URL: Homepage, https://uipath.com
 Project-URL: Repository, https://github.com/UiPath/uipath-python
@@ -17,9 +17,12 @@ Requires-Python: >=3.10
 Requires-Dist: azure-monitor-opentelemetry>=1.6.8
 Requires-Dist: click>=8.1.8
 Requires-Dist: httpx>=0.28.1
+Requires-Dist: hydra-core>=1.3.2
+Requires-Dist: mockito>=1.5.4
 Requires-Dist: opentelemetry-instrumentation>=0.52b1
 Requires-Dist: opentelemetry-sdk>=1.31.1
 Requires-Dist: pathlib>=1.0.1
+Requires-Dist: pydantic-function-models>=0.1.10
 Requires-Dist: pydantic>=2.11.1
 Requires-Dist: pyperclip>=1.9.0
 Requires-Dist: python-dotenv>=1.0.1

{uipath-2.1.72 → uipath-2.1.74}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.1.72"
+version = "2.1.74"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.10"
@@ -19,6 +19,9 @@ dependencies = [
   "truststore>=0.10.1",
   "textual>=5.3.0",
   "pyperclip>=1.9.0",
+  "mockito>=1.5.4",
+  "hydra-core>=1.3.2",
+  "pydantic-function-models>=0.1.10",
 ]
 classifiers = [
   "Development Status :: 3 - Alpha",

uipath-2.1.74/samples/calculator/evals/eval-sets/default.json

@@ -0,0 +1,67 @@
+{
+  "fileName": "default.json",
+  "id": "default-eval-set-id",
+  "name": "Basic Calculator Evaluation Set",
+  "batchSize": 10,
+  "evaluatorRefs": [
+    "equality"
+  ],
+  "evaluations": [
+    {
+      "id": "test-addition",
+      "name": "Test Addition",
+      "inputs": {"a": 1, "b": 1, "operator":  "+"},
+      "expectedOutput": {"result": 2},
+      "expectedAgentBehavior": "",
+      "evalSetId": "default-eval-set-id",
+      "createdAt": "2025-09-04T18:54:58.378Z",
+      "updatedAt": "2025-09-04T18:55:55.416Z"
+    },
+    {
+      "id": "test-random-addition-using-mockito",
+      "name": "Test Random Addition Using Mockito",
+      "inputs": {"a": 1, "b": 1, "operator":  "random"},
+      "expectedOutput": {"result": 2},
+      "expectedAgentBehavior": "",
+      "mockingStrategy": {
+        "type": "mockito",
+        "behaviors": [
+          {
+            "function": "get_random_operator",
+            "arguments": {
+              "args": [],
+              "kwargs": {}
+            },
+            "then": [
+              {
+                "type": "return",
+                "value": {"result": "+"}
+              }
+            ]
+          }
+        ]
+      },
+      "evalSetId": "default-eval-set-id",
+      "createdAt": "2025-09-04T18:54:58.378Z",
+      "updatedAt": "2025-09-04T18:55:55.416Z"
+    },
+    {
+      "id": "test-random-addition-using-llm",
+      "name": "Test Random Addition Using LLM",
+      "inputs": {"a": 1, "b": 1, "operator":  "random"},
+      "expectedOutput": {"result": 2},
+      "expectedAgentBehavior": "",
+      "mockingStrategy": {
+        "type": "llm",
+        "prompt": "The random operator is '+'.",
+        "toolsToSimulate": [{"name":  "get_random_operator"}]
+      },
+      "evalSetId": "default-eval-set-id",
+      "createdAt": "2025-09-04T18:54:58.378Z",
+      "updatedAt": "2025-09-04T18:55:55.416Z"
+    }
+  ],
+  "modelSettings": [],
+  "createdAt": "2025-09-04T18:54:58.379Z",
+  "updatedAt": "2025-09-04T18:55:55.416Z"
+}

{uipath-2.1.72 → uipath-2.1.74}/samples/calculator/evals/evaluators/equality.json

@@ -5,8 +5,6 @@
     "description": "An evaluator that judges the agent based on expected output.",
     "category": 0,
     "type": 1,
-    "prompt": "",
-    "model": "same-as-agent",
     "targetOutputKey": "*",
     "createdAt": "2025-06-26T17:45:39.651Z",
     "updatedAt": "2025-06-26T17:45:39.651Z"

{uipath-2.1.72 → uipath-2.1.74}/samples/calculator/main.py

@@ -1,9 +1,13 @@
+import random
+
 from pydantic.dataclasses import dataclass
 from enum import Enum
 
 from uipath.tracing import traced
 import logging
 
+from uipath._cli._evals.mocks.mocks import mockable
+
 logger = logging.getLogger(__name__)
 
 class Operator(Enum):
@@ -11,6 +15,7 @@ class Operator(Enum):
     SUBTRACT = "-"
     MULTIPLY = "*"
     DIVIDE = "/"
+    RANDOM = "random"
 
 @dataclass
 class CalculatorInput:
@@ -22,13 +27,28 @@ class CalculatorInput:
 class CalculatorOutput:
     result: float
 
-# use InputTriggerEventArgs when called by UiPath EventTriggers
+@dataclass
+class Wrapper:
+    # Testing nested objects
+    result: Operator
+
+@traced()
+@mockable()
+def get_random_operator() -> Wrapper:
+    """Get a random operator."""
+    return Wrapper(result=random.choice([Operator.ADD, Operator.SUBTRACT, Operator.MULTIPLY, Operator.DIVIDE]))
+
+
 @traced()
-def main(input: CalculatorInput) -> CalculatorOutput:
-    result = 0.0
-    match input.operator:
+async def main(input: CalculatorInput) -> CalculatorOutput:
+    if input.operator == Operator.RANDOM:
+        operator = get_random_operator().result
+    else:
+        operator = input.operator
+    match operator:
         case Operator.ADD: result = input.a + input.b
         case Operator.SUBTRACT: result = input.a - input.b
         case Operator.MULTIPLY: result = input.a * input.b
-        case Operator.DIVIDE: result = input.a / input.b if input.b != 0 else 0
+        case Operator.DIVIDE: result = input.a / input.b if input.b != 0.0 else 0.0
+        case _: raise ValueError("Unknown operator")
     return CalculatorOutput(result=result)

uipath-2.1.74/src/uipath/_cli/_evals/_models/_evaluation_set.py

@@ -0,0 +1,167 @@
+from enum import Enum, IntEnum
+from typing import Annotated, Any, Dict, List, Literal, Optional, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+from pydantic.alias_generators import to_camel
+
+
+class EvaluationSimulationTool(BaseModel):
+    name: str = Field(..., alias="name")
+
+
+class MockingStrategyType(str, Enum):
+    LLM = "llm"
+    MOCKITO = "mockito"
+    UNKNOWN = "unknown"
+
+
+class BaseMockingStrategy(BaseModel):
+    pass
+
+
+class LLMMockingStrategy(BaseMockingStrategy):
+    type: Literal[MockingStrategyType.LLM] = MockingStrategyType.LLM
+    prompt: str = Field(..., alias="prompt")
+    tools_to_simulate: list[EvaluationSimulationTool] = Field(
+        ..., alias="toolsToSimulate"
+    )
+
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+"""
+{
+            "function": "postprocess",
+            "arguments": {
+              "args": [],
+              "kwargs": {"x": 3}
+            },
+            "then": [
+              {
+                "return": 3
+              },
+              {
+                "raise": {
+                  "__target__": "NotImplementedError"
+                }
+              }
+            ]
+          }
+          """
+
+
+class MockingArgument(BaseModel):
+    args: List[Any] = Field(default_factory=lambda: [], alias="args")
+    kwargs: Dict[str, Any] = Field(default_factory=lambda: {}, alias="kwargs")
+
+
+class MockingAnswerType(str, Enum):
+    RETURN = "return"
+    RAISE = "raise"
+
+
+class MockingAnswer(BaseModel):
+    type: MockingAnswerType
+    value: Any = Field(..., alias="value")
+
+
+class MockingBehavior(BaseModel):
+    function: str = Field(..., alias="function")
+    arguments: MockingArgument = Field(..., alias="arguments")
+    then: List[MockingAnswer] = Field(..., alias="then")
+
+
+class MockitoMockingStrategy(BaseMockingStrategy):
+    type: Literal[MockingStrategyType.MOCKITO] = MockingStrategyType.MOCKITO
+    behaviors: List[MockingBehavior] = Field(..., alias="config")
+
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+KnownMockingStrategy = Annotated[
+    Union[LLMMockingStrategy, MockitoMockingStrategy],
+    Field(discriminator="type"),
+]
+
+
+class UnknownMockingStrategy(BaseMockingStrategy):
+    type: str = Field(..., alias="type")
+
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+MockingStrategy = Union[KnownMockingStrategy, UnknownMockingStrategy]
+
+
+def migrate_mocking_strategy(data) -> MockingStrategy:
+    if data.get("simulate_tools") and "tools_to_simulate" in data:
+        return LLMMockingStrategy(
+            **{
+                "prompt": data["simulation_instructions"],
+                "toolsToSimulate": data["tools_to_simulate"],
+            }
+        )
+    else:
+        return UnknownMockingStrategy(type=MockingStrategyType.UNKNOWN)
+
+
+class EvaluationItem(BaseModel):
+    """Individual evaluation item within an evaluation set."""
+
+    model_config = ConfigDict(
+        alias_generator=to_camel, populate_by_name=True, extra="allow"
+    )
+
+    id: str
+    name: str
+    inputs: Dict[str, Any]
+    expected_output: Dict[str, Any]
+    expected_agent_behavior: str = Field(default="", alias="expectedAgentBehavior")
+    eval_set_id: str = Field(alias="evalSetId")
+    created_at: str = Field(alias="createdAt")
+    updated_at: str = Field(alias="updatedAt")
+    mocking_strategy: Optional[MockingStrategy] = Field(
+        default=None,
+        alias="mockingStrategy",
+    )
+
+
+class EvaluationSet(BaseModel):
+    """Complete evaluation set model."""
+
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+
+    id: str
+    file_name: str = Field(..., alias="fileName")
+    evaluator_refs: List[str] = Field(default_factory=list)
+    evaluations: List[EvaluationItem] = Field(default_factory=list)
+    name: str
+    batch_size: int = Field(10, alias="batchSize")
+    timeout_minutes: int = Field(default=20, alias="timeoutMinutes")
+    model_settings: List[Dict[str, Any]] = Field(
+        default_factory=list, alias="modelSettings"
+    )
+    created_at: str = Field(alias="createdAt")
+    updated_at: str = Field(alias="updatedAt")
+
+    def extract_selected_evals(self, eval_ids) -> None:
+        selected_evals: list[EvaluationItem] = []
+        for evaluation in self.evaluations:
+            if evaluation.id in eval_ids:
+                selected_evals.append(evaluation)
+                eval_ids.remove(evaluation.id)
+        if len(eval_ids) > 0:
+            raise ValueError("Unknown evaluation ids: {}".format(eval_ids))
+        self.evaluations = selected_evals
+
+
+class EvaluationStatus(IntEnum):
+    PENDING = 0
+    IN_PROGRESS = 1
+    COMPLETED = 2

{uipath-2.1.72 → uipath-2.1.74}/src/uipath/_cli/_evals/_runtime.py

@@ -35,6 +35,7 @@ from ._models._output import (
     UiPathEvalOutput,
     UiPathEvalRunExecutionOutput,
 )
+from .mocks.mocks import set_evaluation_item
 
 T = TypeVar("T", bound=UiPathBaseRuntime)
 C = TypeVar("C", bound=UiPathRuntimeContext)
@@ -137,6 +138,7 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
             evaluation_set_name=evaluation_set.name, score=0, evaluation_set_results=[]
         )
         for eval_item in evaluation_set.evaluations:
+            set_evaluation_item(eval_item)
             await event_bus.publish(
                 EvaluationEvents.CREATE_EVAL_RUN,
                 EvalRunCreatedEvent(

uipath-2.1.74/src/uipath/_cli/_evals/mocks/__init__.py

@@ -0,0 +1 @@
+"""UiPath mocking framework."""

uipath-2.1.74/src/uipath/_cli/_evals/mocks/llm_mocker.py

@@ -0,0 +1,153 @@
+"""LLM mocker implementation."""
+
+import json
+import logging
+from typing import Any, Callable
+
+from pydantic import BaseModel
+
+from uipath._cli._evals._models._evaluation_set import (
+    EvaluationItem,
+    LLMMockingStrategy,
+)
+from uipath._cli._evals.mocks.mocker import Mocker, R, T, UiPathMockingNoMatcherError
+
+PROMPT = """You are simulating a tool call for automated testing purposes of an Agent.
+Your task is to provide a realistic response for the tool based on its schema, examples, and context.
+
+SELECTED TOOL INFORMATION:
+{toolInfo}
+
+SELECTED TOOL SCHEMA:
+{toolSchema}
+
+SELECTED TOOL EXAMPLES:
+{toolRunExamples}
+
+CHOSEN TOOL INPUT:
+{currentToolInput}
+
+CURRENT AGENT RUN SO FAR:
+{testRunHistory}
+
+HERE IS SOME INFORMATION ABOUT THE AGENT: DO NOT USE THIS INFORMATION TO MAKE THE RESPONSE, BUT RATHER TO UNDERSTAND THE CONTEXT IN WHICH THE TOOL IS BEING USED.
+{agentInfo}
+
+TEST RUN PROCTOR INSTRUCTIONS:
+You will need to simulate a real user's interaction with the tool.  This may require following some run specific instructions. If run instructions are provided, follow them exactly.
+Here are the instructions for this run:
+{testRunProctorInstructions}
+
+Based on the above information, provide a realistic response for this tool call. Your response should:
+1. Match the expected output format according to the tool schema
+2. Be very consistent with how the tool has responded in previous examples. Do no omit fields or properties.
+3. Always include the entire output regardless of token length.
+3. Consider the context of the current test run and the agent being tested.  If the agent is acting on a property, make sure the output includes that property.
+
+Respond ONLY with valid JSON that would be a realistic and completetool response. Do not include any explanations or markdown.
+"""
+
+logger = logging.getLogger(__name__)
+
+
+def pydantic_to_dict_safe(obj: Any) -> Any:
+    """Serialize nested pydantic models to a dict."""
+    if isinstance(obj, BaseModel):
+        # Convert Pydantic model to dict recursively
+        return obj.model_dump(mode="json")
+    elif isinstance(obj, dict):
+        # Recursively convert dict entries
+        return {k: pydantic_to_dict_safe(v) for k, v in obj.items()}
+    elif isinstance(obj, list):
+        # Recursively convert items in lists
+        return [pydantic_to_dict_safe(item) for item in obj]
+    return obj  # Return other types as is
+
+
+class LLMMocker(Mocker):
+    """LLM Based Mocker."""
+
+    def __init__(self, evaluation_item: EvaluationItem):
+        """LLM Mocker constructor."""
+        self.evaluation_item = evaluation_item
+        assert isinstance(self.evaluation_item.mocking_strategy, LLMMockingStrategy)
+
+    async def response(
+        self, func: Callable[[T], R], params: dict[str, Any], *args: T, **kwargs
+    ) -> R:
+        """Respond with mocked response generated by an LLM."""
+        assert isinstance(self.evaluation_item.mocking_strategy, LLMMockingStrategy)
+
+        function_name = params.get("name") or func.__name__
+        if function_name in [
+            x.name for x in self.evaluation_item.mocking_strategy.tools_to_simulate
+        ]:
+            from uipath import UiPath
+            from uipath._services.llm_gateway_service import _cleanup_schema
+
+            llm = UiPath().llm
+            return_type: Any = func.__annotations__.get("return", None)
+            if return_type is None:
+                return_type = Any
+
+            class OutputSchema(BaseModel):
+                response: return_type
+
+            response_format = {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": OutputSchema.__name__.lower(),
+                    "strict": True,
+                    "schema": _cleanup_schema(OutputSchema),
+                },
+            }
+            try:
+                prompt_input: dict[str, Any] = {
+                    "toolRunExamples": [],  # Taken from history. Contains id, input json, output json
+                    "testRunHistory": [],  # This should contain ordered spans.
+                    "toolInfo": {
+                        "name": function_name,
+                        "description": params.get("description"),
+                        "arguments": params.get(
+                            "arguments"
+                        ),  # arguments could be passed into tool
+                        "settings": params.get(
+                            "settings"
+                        ),  # settings could be passed into tool
+                        "inputSchema": params.get("input_schema"),
+                    },
+                    "toolSchema": params.get("input_schema"),
+                    "currentToolInput": {
+                        "args": args,
+                        "kwargs": kwargs,
+                    },
+                    "agentInfo": {  # This is incomplete
+                        # "agentName": self.evaluation_item.name, # to be obtained.
+                        "actionName": self.evaluation_item.name,  # Not sure if this is right?
+                        "userInput": self.evaluation_item.inputs,
+                    },
+                    "testRunProctorInstructions": self.evaluation_item.mocking_strategy.prompt,
+                }
+                prompt_input = {
+                    k: json.dumps(pydantic_to_dict_safe(v))
+                    for k, v in prompt_input.items()
+                }
+                response = await llm.chat_completions(
+                    [
+                        {
+                            "role": "user",
+                            "content": PROMPT.format(**prompt_input),
+                        },
+                    ],
+                    response_format=response_format,
+                )
+                mocked_response = OutputSchema(
+                    **json.loads(response.choices[0].message.content)
+                )
+                return mocked_response.response
+            except Exception:
+                raise
+        else:
+            raise UiPathMockingNoMatcherError(
+                f"Method '{function_name}' is not simulated."
+            )

uipath-2.1.74/src/uipath/_cli/_evals/mocks/mocker.py

@@ -0,0 +1,29 @@
+"""Mocker definitions and implementations."""
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from typing import Any, TypeVar
+
+T = TypeVar("T")
+R = TypeVar("R")
+
+
+class Mocker(ABC):
+    """Mocker interface."""
+
+    @abstractmethod
+    async def response(
+        self,
+        func: Callable[[T], R],
+        params: dict[str, Any],
+        *args: T,
+        **kwargs,
+    ) -> R:
+        """Respond with mocked response."""
+        raise NotImplementedError()
+
+
+class UiPathMockingNoMatcherError(Exception):
+    """Exception when a mocker is unable to find a match with the invocation."""
+
+    pass

uipath-2.1.74/src/uipath/_cli/_evals/mocks/mocker_factory.py

@@ -0,0 +1,25 @@
+"""Mocker Factory."""
+
+from uipath._cli._evals._models._evaluation_set import (
+    EvaluationItem,
+    LLMMockingStrategy,
+    MockitoMockingStrategy,
+)
+from uipath._cli._evals.mocks.llm_mocker import LLMMocker
+from uipath._cli._evals.mocks.mocker import Mocker
+from uipath._cli._evals.mocks.mockito_mocker import MockitoMocker
+
+
+class MockerFactory:
+    """Mocker factory."""
+
+    @staticmethod
+    def create(evaluation_item: EvaluationItem) -> Mocker:
+        """Create a mocker instance."""
+        match evaluation_item.mocking_strategy:
+            case LLMMockingStrategy():
+                return LLMMocker(evaluation_item)
+            case MockitoMockingStrategy():
+                return MockitoMocker(evaluation_item)
+            case _:
+                raise ValueError("Unknown mocking strategy")

uipath-2.1.74/src/uipath/_cli/_evals/mocks/mockito_mocker.py

@@ -0,0 +1,62 @@
+"""Mockito mocker implementation.
+
+https://mockito-python.readthedocs.io/en/latest/
+"""
+
+from typing import Any, Callable
+
+from hydra.utils import instantiate
+from mockito import invocation, mocking  # type: ignore[import-untyped]
+
+from uipath._cli._evals._models._evaluation_set import (
+    EvaluationItem,
+    MockingAnswerType,
+    MockitoMockingStrategy,
+)
+from uipath._cli._evals.mocks.mocker import Mocker, R, T
+
+
+class Stub:
+    """Stub interface."""
+
+    def __getattr__(self, item):
+        """Return a wrapper function that raises an exception."""
+
+        def func(*_args, **_kwargs):
+            """Not Implemented."""
+            raise NotImplementedError()
+
+        return func
+
+
+class MockitoMocker(Mocker):
+    """Mockito Mocker."""
+
+    def __init__(self, evaluation_item: EvaluationItem):
+        """Instantiate a mockito mocker."""
+        self.evaluation_item = evaluation_item
+        assert isinstance(self.evaluation_item.mocking_strategy, MockitoMockingStrategy)
+
+        self.stub = Stub()
+        mock_obj = mocking.Mock(self.stub)
+
+        for behavior in self.evaluation_item.mocking_strategy.behaviors:
+            stubbed = invocation.StubbedInvocation(mock_obj, behavior.function)(
+                *instantiate(behavior.arguments.args, _convert_="object"),
+                **instantiate(behavior.arguments.kwargs, _convert_="object"),
+            )
+            for answer in behavior.then:
+                if answer.type == MockingAnswerType.RETURN:
+                    stubbed = stubbed.thenReturn(
+                        instantiate(answer.model_dump(), _convert_="object")["value"]
+                    )
+                elif answer.type == MockingAnswerType.RAISE:
+                    stubbed = stubbed.thenRaise(
+                        instantiate(answer.model_dump(), _convert_="object")["value"]
+                    )
+
+    async def response(
+        self, func: Callable[[T], R], params: dict[str, Any], *args: T, **kwargs
+    ) -> R:
+        """Respond with mocked response."""
+        return getattr(self.stub, params["name"])(*args, **kwargs)