uipath 2.1.76__tar.gz → 2.1.78__tar.gz

{uipath-2.1.76 → uipath-2.1.78}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: uipath
-Version: 2.1.76
+Version: 2.1.78
 Summary: Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools.
 Project-URL: Homepage, https://uipath.com
 Project-URL: Repository, https://github.com/UiPath/uipath-python

{uipath-2.1.76 → uipath-2.1.78}/docs/cli/index.md

@@ -6,6 +6,19 @@
     :depth: 1
     :style: table
 
+/// info | UiPath Automation Suite
+For UiPath Automation Suite deployments, you must set the `UIPATH_URL` environment variable to your dedicated instance URL before running this command.
+
+Example:
+```bash
+UIPATH_URL=https://your-instance.com/account/tenant/orchestrator_/
+```
+
+You can set this environment variable either:
+- In a `.env` file in your project directory
+- As a system-wide environment variable
+///
+
 <!-- termynal -->
 
 ```shell

{uipath-2.1.76 → uipath-2.1.78}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.1.76"
+version = "2.1.78"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.10"

{uipath-2.1.76 → uipath-2.1.78}/samples/calculator/evals/eval-sets/default.json

@@ -55,7 +55,11 @@
       "mockingStrategy": {
         "type": "llm",
         "prompt": "The random operator is '+'.",
-        "toolsToSimulate": [{"name":  "get_random_operator"}]
+        "toolsToSimulate": [{"name":  "get_random_operator"}],
+        "model": {
+          "model": "gpt-4o-mini-2024-07-18",
+          "temperature": 0
+        }
       },
       "evalSetId": "default-eval-set-id",
       "createdAt": "2025-09-04T18:54:58.378Z",

{uipath-2.1.76 → uipath-2.1.78}/samples/calculator/main.py

@@ -3,11 +3,10 @@ import random
 from pydantic.dataclasses import dataclass
 from enum import Enum
 
+from uipath.eval.mocks import mockable, ExampleCall
 from uipath.tracing import traced
 import logging
 
-from uipath._cli._evals.mocks.mocks import mockable
-
 logger = logging.getLogger(__name__)
 
 class Operator(Enum):
@@ -32,8 +31,10 @@ class Wrapper:
     # Testing nested objects
     result: Operator
 
+GET_RANDOM_OPERATOR_EXAMPLES = [ExampleCall(id="example", input="{}", output="{\"result\": \"*\"}")]
+
 @traced()
-@mockable()
+@mockable(example_calls=GET_RANDOM_OPERATOR_EXAMPLES)
 def get_random_operator() -> Wrapper:
     """Get a random operator."""
     return Wrapper(result=random.choice([Operator.ADD, Operator.SUBTRACT, Operator.MULTIPLY, Operator.DIVIDE]))

{uipath-2.1.76 → uipath-2.1.78}/src/uipath/_cli/_evals/_models/_evaluation_set.py

@@ -19,39 +19,31 @@ class BaseMockingStrategy(BaseModel):
     pass
 
 
+class ModelSettings(BaseModel):
+    """Model Generation Parameters."""
+
+    model: str = Field(..., alias="model")
+    temperature: Optional[float] = Field(default=None, alias="temperature")
+    top_p: Optional[float] = Field(default=None, alias="topP")
+    top_k: Optional[int] = Field(default=None, alias="topK")
+    frequency_penalty: Optional[float] = Field(default=None, alias="frequencyPenalty")
+    presence_penalty: Optional[float] = Field(default=None, alias="presencePenalty")
+    max_tokens: Optional[int] = Field(default=None, alias="maxTokens")
+
+
 class LLMMockingStrategy(BaseMockingStrategy):
     type: Literal[MockingStrategyType.LLM] = MockingStrategyType.LLM
     prompt: str = Field(..., alias="prompt")
     tools_to_simulate: list[EvaluationSimulationTool] = Field(
         ..., alias="toolsToSimulate"
     )
+    model: Optional[ModelSettings] = Field(None, alias="model")
 
     model_config = ConfigDict(
         validate_by_name=True, validate_by_alias=True, extra="allow"
     )
 
 
-"""
-{
-            "function": "postprocess",
-            "arguments": {
-              "args": [],
-              "kwargs": {"x": 3}
-            },
-            "then": [
-              {
-                "return": 3
-              },
-              {
-                "raise": {
-                  "__target__": "NotImplementedError"
-                }
-              }
-            ]
-          }
-          """
-
-
 class MockingArgument(BaseModel):
     args: List[Any] = Field(default_factory=lambda: [], alias="args")
     kwargs: Dict[str, Any] = Field(default_factory=lambda: {}, alias="kwargs")
@@ -99,18 +91,6 @@ class UnknownMockingStrategy(BaseMockingStrategy):
 MockingStrategy = Union[KnownMockingStrategy, UnknownMockingStrategy]
 
 
-def migrate_mocking_strategy(data) -> MockingStrategy:
-    if data.get("simulate_tools") and "tools_to_simulate" in data:
-        return LLMMockingStrategy(
-            **{
-                "prompt": data["simulation_instructions"],
-                "toolsToSimulate": data["tools_to_simulate"],
-            }
-        )
-    else:
-        return UnknownMockingStrategy(type=MockingStrategyType.UNKNOWN)
-
-
 class EvaluationItem(BaseModel):
     """Individual evaluation item within an evaluation set."""
 

uipath-2.1.78/src/uipath/_cli/_evals/_models/_mocks.py

@@ -0,0 +1,9 @@
+from pydantic import BaseModel, Field
+
+
+class ExampleCall(BaseModel):
+    """Example invocation for a resource."""
+
+    id: str = Field(..., alias="id")
+    input: str = Field(..., alias="input")
+    output: str = Field(..., alias="output")

{uipath-2.1.76 → uipath-2.1.78}/src/uipath/_cli/_evals/mocks/llm_mocker.py

@@ -6,11 +6,18 @@ from typing import Any, Callable
 
 from pydantic import BaseModel
 
-from uipath._cli._evals._models._evaluation_set import (
+from .._models._evaluation_set import (
     EvaluationItem,
     LLMMockingStrategy,
 )
-from uipath._cli._evals.mocks.mocker import Mocker, R, T, UiPathMockingNoMatcherError
+from .._models._mocks import ExampleCall
+from .mocker import (
+    Mocker,
+    R,
+    T,
+    UiPathMockResponseGenerationError,
+    UiPathNoMockFoundError,
+)
 
 PROMPT = """You are simulating a tool call for automated testing purposes of an Agent.
 Your task is to provide a realistic response for the tool based on its schema, examples, and context.
@@ -102,8 +109,15 @@ class LLMMocker(Mocker):
                 },
             }
             try:
+                # Safely pull examples from params.
+                example_calls = params.get("example_calls", [])
+                if not isinstance(example_calls, list):
+                    example_calls = []
+                example_calls = [
+                    call for call in example_calls if isinstance(call, ExampleCall)
+                ]
                 prompt_input: dict[str, Any] = {
-                    "toolRunExamples": [],  # Taken from history. Contains id, input json, output json
+                    "toolRunExamples": example_calls,
                     "testRunHistory": [],  # This should contain ordered spans.
                     "toolInfo": {
                         "name": function_name,
@@ -132,6 +146,12 @@ class LLMMocker(Mocker):
                     k: json.dumps(pydantic_to_dict_safe(v))
                     for k, v in prompt_input.items()
                 }
+                model_parameters = self.evaluation_item.mocking_strategy.model
+                completion_kwargs = (
+                    model_parameters.model_dump(by_alias=False, exclude_none=True)
+                    if model_parameters
+                    else {}
+                )
                 response = await llm.chat_completions(
                     [
                         {
@@ -140,14 +160,13 @@ class LLMMocker(Mocker):
                         },
                     ],
                     response_format=response_format,
+                    **completion_kwargs,
                 )
                 mocked_response = OutputSchema(
                     **json.loads(response.choices[0].message.content)
                 )
-                return mocked_response.response
-            except Exception:
-                raise
+                return mocked_response.model_dump(mode="json")["response"]
+            except Exception as e:
+                raise UiPathMockResponseGenerationError() from e
         else:
-            raise UiPathMockingNoMatcherError(
-                f"Method '{function_name}' is not simulated."
-            )
+            raise UiPathNoMockFoundError(f"Method '{function_name}' is not simulated.")

{uipath-2.1.76 → uipath-2.1.78}/src/uipath/_cli/_evals/mocks/mocker.py

@@ -23,7 +23,13 @@ class Mocker(ABC):
         raise NotImplementedError()
 
 
-class UiPathMockingNoMatcherError(Exception):
-    """Exception when a mocker is unable to find a match with the invocation."""
+class UiPathNoMockFoundError(Exception):
+    """Exception when a mocker is unable to find a match with the invocation. This is a signal to invoke the real function."""
+
+    pass
+
+
+class UiPathMockResponseGenerationError(Exception):
+    """Exception when a mocker is configured unable to generate a response."""
 
     pass

{uipath-2.1.76 → uipath-2.1.78}/src/uipath/_cli/_evals/mocks/mockito_mocker.py

@@ -13,7 +13,13 @@ from uipath._cli._evals._models._evaluation_set import (
     MockingAnswerType,
     MockitoMockingStrategy,
 )
-from uipath._cli._evals.mocks.mocker import Mocker, R, T
+from uipath._cli._evals.mocks.mocker import (
+    Mocker,
+    R,
+    T,
+    UiPathMockResponseGenerationError,
+    UiPathNoMockFoundError,
+)
 
 
 class Stub:
@@ -59,4 +65,16 @@ class MockitoMocker(Mocker):
         self, func: Callable[[T], R], params: dict[str, Any], *args: T, **kwargs
     ) -> R:
         """Respond with mocked response."""
-        return getattr(self.stub, params["name"])(*args, **kwargs)
+        if not isinstance(
+            self.evaluation_item.mocking_strategy, MockitoMockingStrategy
+        ):
+            raise UiPathMockResponseGenerationError("Mocking strategy misconfigured.")
+        if not any(
+            behavior.function == params["name"]
+            for behavior in self.evaluation_item.mocking_strategy.behaviors
+        ):
+            raise UiPathNoMockFoundError()
+        try:
+            return getattr(self.stub, params["name"])(*args, **kwargs)
+        except Exception as e:
+            raise UiPathMockResponseGenerationError() from e

uipath-2.1.78/src/uipath/_cli/_evals/mocks/mocks.py

@@ -0,0 +1,41 @@
+"""Mocking interface."""
+
+import logging
+from contextvars import ContextVar
+from typing import Any, Callable, Optional
+
+from uipath._cli._evals._models._evaluation_set import EvaluationItem
+from uipath._cli._evals.mocks.mocker import Mocker, UiPathNoMockFoundError
+from uipath._cli._evals.mocks.mocker_factory import MockerFactory
+
+evaluation_context: ContextVar[Optional[EvaluationItem]] = ContextVar(
+    "evaluation", default=None
+)
+
+mocker_context: ContextVar[Optional[Mocker]] = ContextVar("mocker", default=None)
+
+logger = logging.getLogger(__name__)
+
+
+def set_evaluation_item(item: EvaluationItem) -> None:
+    """Set an evaluation item within an evaluation set."""
+    evaluation_context.set(item)
+    try:
+        if item.mocking_strategy:
+            mocker_context.set(MockerFactory.create(item))
+        else:
+            mocker_context.set(None)
+    except Exception:
+        logger.warning(f"Failed to create mocker for evaluation {item.name}")
+        mocker_context.set(None)
+
+
+async def get_mocked_response(
+    func: Callable[[Any], Any], params: dict[str, Any], *args, **kwargs
+) -> Any:
+    """Get a mocked response."""
+    mocker = mocker_context.get()
+    if mocker is None:
+        raise UiPathNoMockFoundError()
+    else:
+        return await mocker.response(func, params, *args, **kwargs)

{uipath-2.1.76 → uipath-2.1.78}/src/uipath/_services/llm_gateway_service.py

@@ -351,7 +351,8 @@ class UiPathLlmChatService(BaseService):
         n: int = 1,
         frequency_penalty: float = 0,
         presence_penalty: float = 0,
-        top_p: float = 1,
+        top_p: Optional[float] = 1,
+        top_k: Optional[int] = None,
         tools: Optional[List[ToolDefinition]] = None,
         tool_choice: Optional[ToolChoice] = None,
         response_format: Optional[Union[Dict[str, Any], type[BaseModel]]] = None,
@@ -385,6 +386,8 @@ class UiPathLlmChatService(BaseService):
                 Positive values encourage discussion of new topics. Defaults to 0.
             top_p (float, optional): Nucleus sampling parameter between 0 and 1.
                 Controls diversity by considering only the top p probability mass. Defaults to 1.
+            top_k (int, optional): Nucleus sampling parameter.
+                Controls diversity by considering only the top k most probable tokens. Defaults to None.
             tools (Optional[List[ToolDefinition]], optional): List of tool definitions that the
                 model can call. Tools enable the model to perform actions or retrieve information
                 beyond text generation. Defaults to None.
@@ -486,6 +489,8 @@ class UiPathLlmChatService(BaseService):
             "presence_penalty": presence_penalty,
             "top_p": top_p,
         }
+        if top_k is not None:
+            request_body["top_k"] = top_k
 
         # Handle response_format - convert BaseModel to schema if needed
         if response_format:

{uipath-2.1.76 → uipath-2.1.78}/src/uipath/agent/models/agent.py

@@ -7,6 +7,7 @@ from pydantic import BaseModel, ConfigDict, Discriminator, Field, Tag
 
 from uipath._cli._evals._models._evaluation_set import EvaluationSet
 from uipath._cli._evals._models._evaluator import Evaluator
+from uipath._cli._evals._models._mocks import ExampleCall
 from uipath.models import Connection
 
 
@@ -70,7 +71,13 @@ class AgentToolSettings(BaseModel):
     )
 
 
-class AgentProcessToolProperties(BaseModel):
+class BaseResourceProperties(BaseModel):
+    """Base resource properties."""
+
+    example_calls: Optional[List[ExampleCall]] = Field(None, alias="exampleCalls")
+
+
+class AgentProcessToolProperties(BaseResourceProperties):
     """Properties specific to tool configuration."""
 
     folder_path: Optional[str] = Field(None, alias="folderPath")
@@ -125,7 +132,7 @@ class AgentIntegrationToolParameter(BaseModel):
     )
 
 
-class AgentIntegrationToolProperties(BaseModel):
+class AgentIntegrationToolProperties(BaseResourceProperties):
     """Properties specific to tool configuration."""
 
     tool_path: str = Field(..., alias="toolPath")
@@ -198,7 +205,7 @@ class AgentEscalationRecipient(BaseModel):
     )
 
 
-class AgentEscalationChannelProperties(BaseModel):
+class AgentEscalationChannelProperties(BaseResourceProperties):
     """Agent escalation channel properties."""
 
     app_name: str = Field(..., alias="appName")

{uipath-2.1.76 → uipath-2.1.78}/src/uipath/eval/evaluators/trajectory_evaluator.py

@@ -6,12 +6,16 @@ from typing import Any, Optional
 from opentelemetry.sdk.trace import ReadableSpan
 from pydantic import field_validator
 
-from uipath._cli._evals._models._trajectory_span import TrajectoryEvaluationTrace
 from uipath.eval.models import EvaluationResult
 
 from ..._services import UiPathLlmChatService
 from ..._utils.constants import COMMUNITY_agents_SUFFIX
-from ..models.models import AgentExecution, LLMResponse, NumericEvaluationResult
+from ..models.models import (
+    AgentExecution,
+    LLMResponse,
+    NumericEvaluationResult,
+    TrajectoryEvaluationTrace,
+)
 from .base_evaluator import BaseEvaluator
 
 

uipath-2.1.78/src/uipath/eval/mocks/__init__.py

@@ -0,0 +1,7 @@
+"""Mock interface."""
+
+from uipath._cli._evals._models._mocks import ExampleCall
+
+from .mockable import mockable
+
+__all__ = ["ExampleCall", "mockable"]

uipath-2.1.76/src/uipath/_cli/_evals/mocks/mocks.py → uipath-2.1.78/src/uipath/eval/mocks/mockable.py

@@ -1,49 +1,18 @@
-"""Mocking interface."""
+"""Mockable interface."""
 
 import asyncio
 import functools
 import inspect
 import logging
 import threading
-from contextvars import ContextVar
-from typing import Any, Callable, Optional
+from typing import Any, List, Optional
 
 from pydantic import TypeAdapter
 from pydantic_function_models import ValidatedFunction  # type: ignore[import-untyped]
 
-from uipath._cli._evals._models._evaluation_set import EvaluationItem
-from uipath._cli._evals.mocks.mocker import Mocker
-from uipath._cli._evals.mocks.mocker_factory import MockerFactory
-
-evaluation_context: ContextVar[Optional[EvaluationItem]] = ContextVar(
-    "evaluation", default=None
-)
-
-mocker_context: ContextVar[Optional[Mocker]] = ContextVar("mocker", default=None)
-
-
-def set_evaluation_item(item: EvaluationItem) -> None:
-    """Set an evaluation item within an evaluation set."""
-    evaluation_context.set(item)
-    try:
-        mocker_context.set(MockerFactory.create(item))
-    except Exception:
-        logger.warning(f"Failed to create mocker for evaluation {item.name}")
-        mocker_context.set(None)
-
-
-async def get_mocked_response(
-    func: Callable[[Any], Any], params: dict[str, Any], *args, **kwargs
-) -> Any:
-    """Get a mocked response."""
-    mocker = mocker_context.get()
-    evaluation_item = evaluation_context.get()
-    if mocker is None or evaluation_item is None:
-        # TODO raise a new UiPath exception type
-        raise RuntimeError(f"Evaluation item {func.__name__} has not been evaluated")
-    else:
-        return await mocker.response(func, params, *args, **kwargs)
-
+from uipath._cli._evals._models._mocks import ExampleCall
+from uipath._cli._evals.mocks.mocker import UiPathNoMockFoundError
+from uipath._cli._evals.mocks.mocks import get_mocked_response
 
 _event_loop = None
 logger = logging.getLogger(__name__)
@@ -77,10 +46,7 @@ def mocked_response_decorator(func, params: dict[str, Any]):
         async def decorated_func(*args, **kwargs):
             try:
                 return await mock_response_generator(*args, **kwargs)
-            except Exception:
-                logger.warning(
-                    f"Failed to mock response for {func.__name__}. Falling back to func."
-                )
+            except UiPathNoMockFoundError:
                 return await func(*args, **kwargs)
     else:
 
@@ -88,18 +54,35 @@ def mocked_response_decorator(func, params: dict[str, Any]):
         def decorated_func(*args, **kwargs):
             try:
                 return run_coroutine(mock_response_generator(*args, **kwargs))
-            except Exception:
-                logger.warning(
-                    f"Failed to mock response for {func.__name__}. Falling back to func."
-                )
+            except UiPathNoMockFoundError:
                 return func(*args, **kwargs)
 
     return decorated_func
 
 
+def get_output_schema(func):
+    """Retrieves the JSON schema for a function's return type hint."""
+    try:
+        adapter = TypeAdapter(inspect.signature(func).return_annotation)
+        return adapter.json_schema()
+    except Exception:
+        logger.warning(f"Unable to extract output schema for function {func.__name__}")
+        return {}
+
+
+def get_input_schema(func):
+    """Retrieves the JSON schema for a function's input type."""
+    try:
+        return ValidatedFunction(func).model.model_json_schema()
+    except Exception:
+        logger.warning(f"Unable to extract input schema for function {func.__name__}")
+        return {}
+
+
 def mockable(
     name: Optional[str] = None,
     description: Optional[str] = None,
+    example_calls: Optional[List[ExampleCall]] = None,
     **kwargs,
 ):
     """Decorate a function to be a mockable."""
@@ -110,27 +93,9 @@ def mockable(
             "description": description or func.__doc__,
             "input_schema": get_input_schema(func),
             "output_schema": get_output_schema(func),
+            "example_calls": example_calls,
             **kwargs,
         }
         return mocked_response_decorator(func, params)
 
     return decorator
-
-
-def get_output_schema(func):
-    """Retrieves the JSON schema for a function's return type hint."""
-    try:
-        adapter = TypeAdapter(inspect.signature(func).return_annotation)
-        return adapter.json_schema()
-    except Exception:
-        logger.warning(f"Unable to extract output schema for function {func.__name__}")
-        return {}
-
-
-def get_input_schema(func):
-    """Retrieves the JSON schema for a function's input type."""
-    try:
-        return ValidatedFunction(func).model.model_json_schema()
-    except Exception:
-        logger.warning(f"Unable to extract input schema for function {func.__name__}")
-        return {}

uipath-2.1.76/src/uipath/_cli/_evals/_models/_trajectory_span.py → uipath-2.1.78/src/uipath/eval/models/models.py

@@ -1,10 +1,119 @@
-"""Trajectory evaluation span model for serializing span data in evaluations."""
+"""Models for evaluation framework including execution data and evaluation results."""
 
 from dataclasses import dataclass
-from typing import Any, Dict, List, Optional
+from enum import IntEnum
+from typing import Annotated, Any, Dict, List, Literal, Optional, Union
 
 from opentelemetry.sdk.trace import ReadableSpan
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class AgentExecution(BaseModel):
+    """Represents the execution data of an agent for evaluation purposes."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    agent_input: Optional[Dict[str, Any]]
+    agent_output: Dict[str, Any]
+    agent_trace: list[ReadableSpan]
+    expected_agent_behavior: Optional[str] = None
+
+
+class LLMResponse(BaseModel):
+    """Response from an LLM evaluator."""
+
+    score: float
+    justification: str
+
+
+class ScoreType(IntEnum):
+    """Types of evaluation scores."""
+
+    BOOLEAN = 0
+    NUMERICAL = 1
+    ERROR = 2
+
+
+class BaseEvaluationResult(BaseModel):
+    """Base class for evaluation results."""
+
+    details: Optional[str] = None
+    # this is marked as optional, as it is populated inside the 'measure_execution_time' decorator
+    evaluation_time: Optional[float] = None
+
+
+class BooleanEvaluationResult(BaseEvaluationResult):
+    """Result of a boolean evaluation."""
+
+    score: bool
+    score_type: Literal[ScoreType.BOOLEAN] = ScoreType.BOOLEAN
+
+
+class NumericEvaluationResult(BaseEvaluationResult):
+    """Result of a numerical evaluation."""
+
+    score: float
+    score_type: Literal[ScoreType.NUMERICAL] = ScoreType.NUMERICAL
+
+
+class ErrorEvaluationResult(BaseEvaluationResult):
+    """Result of an error evaluation."""
+
+    score: float = 0.0
+    score_type: Literal[ScoreType.ERROR] = ScoreType.ERROR
+
+
+EvaluationResult = Annotated[
+    Union[BooleanEvaluationResult, NumericEvaluationResult, ErrorEvaluationResult],
+    Field(discriminator="score_type"),
+]
+
+
+class EvalItemResult(BaseModel):
+    """Result of a single evaluation item."""
+
+    evaluator_id: str
+    result: EvaluationResult
+
+
+class EvaluatorCategory(IntEnum):
+    """Types of evaluators."""
+
+    Deterministic = 0
+    LlmAsAJudge = 1
+    AgentScorer = 2
+    Trajectory = 3
+
+    @classmethod
+    def from_int(cls, value):
+        """Construct EvaluatorCategory from an int value."""
+        if value in cls._value2member_map_:
+            return cls(value)
+        else:
+            raise ValueError(f"{value} is not a valid EvaluatorCategory value")
+
+
+class EvaluatorType(IntEnum):
+    """Subtypes of evaluators."""
+
+    Unknown = 0
+    Equals = 1
+    Contains = 2
+    Regex = 3
+    Factuality = 4
+    Custom = 5
+    JsonSimilarity = 6
+    Trajectory = 7
+    ContextPrecision = 8
+    Faithfulness = 9
+
+    @classmethod
+    def from_int(cls, value):
+        """Construct EvaluatorCategory from an int value."""
+        if value in cls._value2member_map_:
+            return cls(value)
+        else:
+            raise ValueError(f"{value} is not a valid EvaluatorType value")
 
 
 @dataclass