uipath 2.1.72__py3-none-any.whl → 2.1.74__py3-none-any.whl

uipath/_cli/_evals/_models/_evaluation_set.py

@@ -1,5 +1,5 @@
-from enum import IntEnum
-from typing import Any, Dict, List
+from enum import Enum, IntEnum
+from typing import Annotated, Any, Dict, List, Literal, Optional, Union
 
 from pydantic import BaseModel, ConfigDict, Field
 from pydantic.alias_generators import to_camel
@@ -9,28 +9,127 @@ class EvaluationSimulationTool(BaseModel):
     name: str = Field(..., alias="name")
 
 
+class MockingStrategyType(str, Enum):
+    LLM = "llm"
+    MOCKITO = "mockito"
+    UNKNOWN = "unknown"
+
+
+class BaseMockingStrategy(BaseModel):
+    pass
+
+
+class LLMMockingStrategy(BaseMockingStrategy):
+    type: Literal[MockingStrategyType.LLM] = MockingStrategyType.LLM
+    prompt: str = Field(..., alias="prompt")
+    tools_to_simulate: list[EvaluationSimulationTool] = Field(
+        ..., alias="toolsToSimulate"
+    )
+
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+"""
+{
+            "function": "postprocess",
+            "arguments": {
+              "args": [],
+              "kwargs": {"x": 3}
+            },
+            "then": [
+              {
+                "return": 3
+              },
+              {
+                "raise": {
+                  "__target__": "NotImplementedError"
+                }
+              }
+            ]
+          }
+          """
+
+
+class MockingArgument(BaseModel):
+    args: List[Any] = Field(default_factory=lambda: [], alias="args")
+    kwargs: Dict[str, Any] = Field(default_factory=lambda: {}, alias="kwargs")
+
+
+class MockingAnswerType(str, Enum):
+    RETURN = "return"
+    RAISE = "raise"
+
+
+class MockingAnswer(BaseModel):
+    type: MockingAnswerType
+    value: Any = Field(..., alias="value")
+
+
+class MockingBehavior(BaseModel):
+    function: str = Field(..., alias="function")
+    arguments: MockingArgument = Field(..., alias="arguments")
+    then: List[MockingAnswer] = Field(..., alias="then")
+
+
+class MockitoMockingStrategy(BaseMockingStrategy):
+    type: Literal[MockingStrategyType.MOCKITO] = MockingStrategyType.MOCKITO
+    behaviors: List[MockingBehavior] = Field(..., alias="config")
+
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+KnownMockingStrategy = Annotated[
+    Union[LLMMockingStrategy, MockitoMockingStrategy],
+    Field(discriminator="type"),
+]
+
+
+class UnknownMockingStrategy(BaseMockingStrategy):
+    type: str = Field(..., alias="type")
+
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+MockingStrategy = Union[KnownMockingStrategy, UnknownMockingStrategy]
+
+
+def migrate_mocking_strategy(data) -> MockingStrategy:
+    if data.get("simulate_tools") and "tools_to_simulate" in data:
+        return LLMMockingStrategy(
+            **{
+                "prompt": data["simulation_instructions"],
+                "toolsToSimulate": data["tools_to_simulate"],
+            }
+        )
+    else:
+        return UnknownMockingStrategy(type=MockingStrategyType.UNKNOWN)
+
+
 class EvaluationItem(BaseModel):
     """Individual evaluation item within an evaluation set."""
 
-    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+    model_config = ConfigDict(
+        alias_generator=to_camel, populate_by_name=True, extra="allow"
+    )
 
     id: str
     name: str
     inputs: Dict[str, Any]
     expected_output: Dict[str, Any]
     expected_agent_behavior: str = Field(default="", alias="expectedAgentBehavior")
-    simulation_instructions: str = Field(default="", alias="simulationInstructions")
-    simulate_input: bool = Field(default=False, alias="simulateInput")
-    input_generation_instructions: str = Field(
-        default="", alias="inputGenerationInstructions"
-    )
-    simulate_tools: bool = Field(default=False, alias="simulateTools")
-    tools_to_simulate: List[EvaluationSimulationTool] = Field(
-        default_factory=list, alias="toolsToSimulate"
-    )
     eval_set_id: str = Field(alias="evalSetId")
     created_at: str = Field(alias="createdAt")
     updated_at: str = Field(alias="updatedAt")
+    mocking_strategy: Optional[MockingStrategy] = Field(
+        default=None,
+        alias="mockingStrategy",
+    )
 
 
 class EvaluationSet(BaseModel):

uipath/_cli/_evals/_runtime.py

@@ -35,6 +35,7 @@ from ._models._output import (
     UiPathEvalOutput,
     UiPathEvalRunExecutionOutput,
 )
+from .mocks.mocks import set_evaluation_item
 
 T = TypeVar("T", bound=UiPathBaseRuntime)
 C = TypeVar("C", bound=UiPathRuntimeContext)
@@ -137,6 +138,7 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
             evaluation_set_name=evaluation_set.name, score=0, evaluation_set_results=[]
         )
         for eval_item in evaluation_set.evaluations:
+            set_evaluation_item(eval_item)
             await event_bus.publish(
                 EvaluationEvents.CREATE_EVAL_RUN,
                 EvalRunCreatedEvent(

uipath/_cli/_evals/mocks/__init__.py

@@ -0,0 +1 @@
+"""UiPath mocking framework."""

uipath/_cli/_evals/mocks/llm_mocker.py

@@ -0,0 +1,153 @@
+"""LLM mocker implementation."""
+
+import json
+import logging
+from typing import Any, Callable
+
+from pydantic import BaseModel
+
+from uipath._cli._evals._models._evaluation_set import (
+    EvaluationItem,
+    LLMMockingStrategy,
+)
+from uipath._cli._evals.mocks.mocker import Mocker, R, T, UiPathMockingNoMatcherError
+
+PROMPT = """You are simulating a tool call for automated testing purposes of an Agent.
+Your task is to provide a realistic response for the tool based on its schema, examples, and context.
+
+SELECTED TOOL INFORMATION:
+{toolInfo}
+
+SELECTED TOOL SCHEMA:
+{toolSchema}
+
+SELECTED TOOL EXAMPLES:
+{toolRunExamples}
+
+CHOSEN TOOL INPUT:
+{currentToolInput}
+
+CURRENT AGENT RUN SO FAR:
+{testRunHistory}
+
+HERE IS SOME INFORMATION ABOUT THE AGENT: DO NOT USE THIS INFORMATION TO MAKE THE RESPONSE, BUT RATHER TO UNDERSTAND THE CONTEXT IN WHICH THE TOOL IS BEING USED.
+{agentInfo}
+
+TEST RUN PROCTOR INSTRUCTIONS:
+You will need to simulate a real user's interaction with the tool.  This may require following some run specific instructions. If run instructions are provided, follow them exactly.
+Here are the instructions for this run:
+{testRunProctorInstructions}
+
+Based on the above information, provide a realistic response for this tool call. Your response should:
+1. Match the expected output format according to the tool schema
+2. Be very consistent with how the tool has responded in previous examples. Do no omit fields or properties.
+3. Always include the entire output regardless of token length.
+3. Consider the context of the current test run and the agent being tested.  If the agent is acting on a property, make sure the output includes that property.
+
+Respond ONLY with valid JSON that would be a realistic and completetool response. Do not include any explanations or markdown.
+"""
+
+logger = logging.getLogger(__name__)
+
+
+def pydantic_to_dict_safe(obj: Any) -> Any:
+    """Serialize nested pydantic models to a dict."""
+    if isinstance(obj, BaseModel):
+        # Convert Pydantic model to dict recursively
+        return obj.model_dump(mode="json")
+    elif isinstance(obj, dict):
+        # Recursively convert dict entries
+        return {k: pydantic_to_dict_safe(v) for k, v in obj.items()}
+    elif isinstance(obj, list):
+        # Recursively convert items in lists
+        return [pydantic_to_dict_safe(item) for item in obj]
+    return obj  # Return other types as is
+
+
+class LLMMocker(Mocker):
+    """LLM Based Mocker."""
+
+    def __init__(self, evaluation_item: EvaluationItem):
+        """LLM Mocker constructor."""
+        self.evaluation_item = evaluation_item
+        assert isinstance(self.evaluation_item.mocking_strategy, LLMMockingStrategy)
+
+    async def response(
+        self, func: Callable[[T], R], params: dict[str, Any], *args: T, **kwargs
+    ) -> R:
+        """Respond with mocked response generated by an LLM."""
+        assert isinstance(self.evaluation_item.mocking_strategy, LLMMockingStrategy)
+
+        function_name = params.get("name") or func.__name__
+        if function_name in [
+            x.name for x in self.evaluation_item.mocking_strategy.tools_to_simulate
+        ]:
+            from uipath import UiPath
+            from uipath._services.llm_gateway_service import _cleanup_schema
+
+            llm = UiPath().llm
+            return_type: Any = func.__annotations__.get("return", None)
+            if return_type is None:
+                return_type = Any
+
+            class OutputSchema(BaseModel):
+                response: return_type
+
+            response_format = {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": OutputSchema.__name__.lower(),
+                    "strict": True,
+                    "schema": _cleanup_schema(OutputSchema),
+                },
+            }
+            try:
+                prompt_input: dict[str, Any] = {
+                    "toolRunExamples": [],  # Taken from history. Contains id, input json, output json
+                    "testRunHistory": [],  # This should contain ordered spans.
+                    "toolInfo": {
+                        "name": function_name,
+                        "description": params.get("description"),
+                        "arguments": params.get(
+                            "arguments"
+                        ),  # arguments could be passed into tool
+                        "settings": params.get(
+                            "settings"
+                        ),  # settings could be passed into tool
+                        "inputSchema": params.get("input_schema"),
+                    },
+                    "toolSchema": params.get("input_schema"),
+                    "currentToolInput": {
+                        "args": args,
+                        "kwargs": kwargs,
+                    },
+                    "agentInfo": {  # This is incomplete
+                        # "agentName": self.evaluation_item.name, # to be obtained.
+                        "actionName": self.evaluation_item.name,  # Not sure if this is right?
+                        "userInput": self.evaluation_item.inputs,
+                    },
+                    "testRunProctorInstructions": self.evaluation_item.mocking_strategy.prompt,
+                }
+                prompt_input = {
+                    k: json.dumps(pydantic_to_dict_safe(v))
+                    for k, v in prompt_input.items()
+                }
+                response = await llm.chat_completions(
+                    [
+                        {
+                            "role": "user",
+                            "content": PROMPT.format(**prompt_input),
+                        },
+                    ],
+                    response_format=response_format,
+                )
+                mocked_response = OutputSchema(
+                    **json.loads(response.choices[0].message.content)
+                )
+                return mocked_response.response
+            except Exception:
+                raise
+        else:
+            raise UiPathMockingNoMatcherError(
+                f"Method '{function_name}' is not simulated."
+            )

uipath/_cli/_evals/mocks/mocker.py

@@ -0,0 +1,29 @@
+"""Mocker definitions and implementations."""
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from typing import Any, TypeVar
+
+T = TypeVar("T")
+R = TypeVar("R")
+
+
+class Mocker(ABC):
+    """Mocker interface."""
+
+    @abstractmethod
+    async def response(
+        self,
+        func: Callable[[T], R],
+        params: dict[str, Any],
+        *args: T,
+        **kwargs,
+    ) -> R:
+        """Respond with mocked response."""
+        raise NotImplementedError()
+
+
+class UiPathMockingNoMatcherError(Exception):
+    """Exception when a mocker is unable to find a match with the invocation."""
+
+    pass

uipath/_cli/_evals/mocks/mocker_factory.py

@@ -0,0 +1,25 @@
+"""Mocker Factory."""
+
+from uipath._cli._evals._models._evaluation_set import (
+    EvaluationItem,
+    LLMMockingStrategy,
+    MockitoMockingStrategy,
+)
+from uipath._cli._evals.mocks.llm_mocker import LLMMocker
+from uipath._cli._evals.mocks.mocker import Mocker
+from uipath._cli._evals.mocks.mockito_mocker import MockitoMocker
+
+
+class MockerFactory:
+    """Mocker factory."""
+
+    @staticmethod
+    def create(evaluation_item: EvaluationItem) -> Mocker:
+        """Create a mocker instance."""
+        match evaluation_item.mocking_strategy:
+            case LLMMockingStrategy():
+                return LLMMocker(evaluation_item)
+            case MockitoMockingStrategy():
+                return MockitoMocker(evaluation_item)
+            case _:
+                raise ValueError("Unknown mocking strategy")

uipath/_cli/_evals/mocks/mockito_mocker.py

@@ -0,0 +1,62 @@
+"""Mockito mocker implementation.
+
+https://mockito-python.readthedocs.io/en/latest/
+"""
+
+from typing import Any, Callable
+
+from hydra.utils import instantiate
+from mockito import invocation, mocking  # type: ignore[import-untyped]
+
+from uipath._cli._evals._models._evaluation_set import (
+    EvaluationItem,
+    MockingAnswerType,
+    MockitoMockingStrategy,
+)
+from uipath._cli._evals.mocks.mocker import Mocker, R, T
+
+
+class Stub:
+    """Stub interface."""
+
+    def __getattr__(self, item):
+        """Return a wrapper function that raises an exception."""
+
+        def func(*_args, **_kwargs):
+            """Not Implemented."""
+            raise NotImplementedError()
+
+        return func
+
+
+class MockitoMocker(Mocker):
+    """Mockito Mocker."""
+
+    def __init__(self, evaluation_item: EvaluationItem):
+        """Instantiate a mockito mocker."""
+        self.evaluation_item = evaluation_item
+        assert isinstance(self.evaluation_item.mocking_strategy, MockitoMockingStrategy)
+
+        self.stub = Stub()
+        mock_obj = mocking.Mock(self.stub)
+
+        for behavior in self.evaluation_item.mocking_strategy.behaviors:
+            stubbed = invocation.StubbedInvocation(mock_obj, behavior.function)(
+                *instantiate(behavior.arguments.args, _convert_="object"),
+                **instantiate(behavior.arguments.kwargs, _convert_="object"),
+            )
+            for answer in behavior.then:
+                if answer.type == MockingAnswerType.RETURN:
+                    stubbed = stubbed.thenReturn(
+                        instantiate(answer.model_dump(), _convert_="object")["value"]
+                    )
+                elif answer.type == MockingAnswerType.RAISE:
+                    stubbed = stubbed.thenRaise(
+                        instantiate(answer.model_dump(), _convert_="object")["value"]
+                    )
+
+    async def response(
+        self, func: Callable[[T], R], params: dict[str, Any], *args: T, **kwargs
+    ) -> R:
+        """Respond with mocked response."""
+        return getattr(self.stub, params["name"])(*args, **kwargs)

uipath/_cli/_evals/mocks/mocks.py

@@ -0,0 +1,136 @@
+"""Mocking interface."""
+
+import asyncio
+import functools
+import inspect
+import logging
+import threading
+from contextvars import ContextVar
+from typing import Any, Callable, Optional
+
+from pydantic import TypeAdapter
+from pydantic_function_models import ValidatedFunction  # type: ignore[import-untyped]
+
+from uipath._cli._evals._models._evaluation_set import EvaluationItem
+from uipath._cli._evals.mocks.mocker import Mocker
+from uipath._cli._evals.mocks.mocker_factory import MockerFactory
+
+evaluation_context: ContextVar[Optional[EvaluationItem]] = ContextVar(
+    "evaluation", default=None
+)
+
+mocker_context: ContextVar[Optional[Mocker]] = ContextVar("mocker", default=None)
+
+
+def set_evaluation_item(item: EvaluationItem) -> None:
+    """Set an evaluation item within an evaluation set."""
+    evaluation_context.set(item)
+    try:
+        mocker_context.set(MockerFactory.create(item))
+    except Exception:
+        logger.warning(f"Failed to create mocker for evaluation {item.name}")
+        mocker_context.set(None)
+
+
+async def get_mocked_response(
+    func: Callable[[Any], Any], params: dict[str, Any], *args, **kwargs
+) -> Any:
+    """Get a mocked response."""
+    mocker = mocker_context.get()
+    evaluation_item = evaluation_context.get()
+    if mocker is None or evaluation_item is None:
+        # TODO raise a new UiPath exception type
+        raise RuntimeError(f"Evaluation item {func.__name__} has not been evaluated")
+    else:
+        return await mocker.response(func, params, *args, **kwargs)
+
+
+_event_loop = None
+logger = logging.getLogger(__name__)
+
+
+def run_coroutine(coro):
+    """Run a coroutine synchronously."""
+    global _event_loop
+    if not _event_loop or not _event_loop.is_running():
+        _event_loop = asyncio.new_event_loop()
+        threading.Thread(target=_event_loop.run_forever, daemon=True).start()
+    future = asyncio.run_coroutine_threadsafe(coro, _event_loop)
+    return future.result()
+
+
+def mocked_response_decorator(func, params: dict[str, Any]):
+    """Mocked response decorator."""
+
+    async def mock_response_generator(*args, **kwargs):
+        mocked_response = await get_mocked_response(func, params, *args, **kwargs)
+        return_type: Any = func.__annotations__.get("return", None)
+
+        if return_type is not None:
+            mocked_response = TypeAdapter(return_type).validate_python(mocked_response)
+        return mocked_response
+
+    is_async = inspect.iscoroutinefunction(func)
+    if is_async:
+
+        @functools.wraps(func)
+        async def decorated_func(*args, **kwargs):
+            try:
+                return await mock_response_generator(*args, **kwargs)
+            except Exception:
+                logger.warning(
+                    f"Failed to mock response for {func.__name__}. Falling back to func."
+                )
+                return await func(*args, **kwargs)
+    else:
+
+        @functools.wraps(func)
+        def decorated_func(*args, **kwargs):
+            try:
+                return run_coroutine(mock_response_generator(*args, **kwargs))
+            except Exception:
+                logger.warning(
+                    f"Failed to mock response for {func.__name__}. Falling back to func."
+                )
+                return func(*args, **kwargs)
+
+    return decorated_func
+
+
+def mockable(
+    name: Optional[str] = None,
+    description: Optional[str] = None,
+    **kwargs,
+):
+    """Decorate a function to be a mockable."""
+
+    def decorator(func):
+        params = {
+            "name": name or func.__name__,
+            "description": description or func.__doc__,
+            "input_schema": get_input_schema(func),
+            "output_schema": get_output_schema(func),
+            **kwargs,
+        }
+        return mocked_response_decorator(func, params)
+
+    return decorator
+
+
+def get_output_schema(func):
+    """Retrieves the JSON schema for a function's return type hint."""
+    try:
+        adapter = TypeAdapter(inspect.signature(func).return_annotation)
+        return adapter.json_schema()
+    except Exception:
+        logger.warning(f"Unable to extract output schema for function {func.__name__}")
+        return {}
+
+
+def get_input_schema(func):
+    """Retrieves the JSON schema for a function's input type."""
+    try:
+        return ValidatedFunction(func).model.model_json_schema()
+    except Exception:
+        logger.warning(f"Unable to extract input schema for function {func.__name__}")
+        return {}

uipath/_cli/_runtime/_logging.py

@@ -50,6 +50,16 @@ class ExecutionContextFilter(logging.Filter):
         return False
 
 
+class MasterExecutionFilter(logging.Filter):
+    """Filter for master handler that blocks logs from any child execution."""
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        """Block logs that belong to a child execution context."""
+        ctx_execution_id = current_execution_id.get()
+        # Block if there's an active child execution context
+        return ctx_execution_id is None
+
+
 class LogsInterceptor:
     """Intercepts all logging and stdout/stderr, routing to either persistent log files or stdout based on whether it's running as a job or not."""
 
@@ -93,7 +103,9 @@ class LogsInterceptor:
         self.original_stderr = cast(TextIO, sys.stderr)
 
         self.log_handler: Union[
-            PersistentLogsHandler, logging.StreamHandler[TextIO], logging.Handler
+            PersistentLogsHandler,
+            logging.StreamHandler[TextIO],
+            logging.Handler,
         ]
 
         if log_handler:
@@ -116,9 +128,14 @@ class LogsInterceptor:
         self.log_handler.setLevel(self.numeric_min_level)
 
         # Add execution context filter if execution_id provided
+        self.execution_filter: Optional[logging.Filter] = None
         if execution_id:
             self.execution_filter = ExecutionContextFilter(execution_id)
             self.log_handler.addFilter(self.execution_filter)
+        else:
+            # Master execution: filter out child execution logs
+            self.execution_filter = MasterExecutionFilter()
+            self.log_handler.addFilter(self.execution_filter)
 
         self.logger = logging.getLogger("runtime")
         self.patched_loggers: set[str] = set()
@@ -146,17 +163,23 @@ class LogsInterceptor:
         self.root_logger.setLevel(self.numeric_min_level)
 
         if self.execution_id:
-            # Parallel execution mode: add our handler without removing others
+            # Child execution mode: add our handler without removing others
             if self.log_handler not in self.root_logger.handlers:
                 self.root_logger.addHandler(self.log_handler)
 
-            # Set up propagation for all existing loggers
+            # Keep propagation enabled so logs flow through filters
+            # Our ExecutionContextFilter will ensure only our logs get through our handler
             for logger_name in logging.root.manager.loggerDict:
                 logger = logging.getLogger(logger_name)
-                # Keep propagation enabled so logs flow to all handlers
+                # Keep propagation enabled for filtering to work
+                # logger.propagate remains True (default)
                 self.patched_loggers.add(logger_name)
+
+            # Child executions should redirect stdout/stderr to their own handler
+            # This ensures print statements are captured per execution
+            self._redirect_stdout_stderr()
         else:
-            # Single execution mode: remove all handlers and add only ours
+            # Master execution mode: remove all handlers and add only ours
             self._clean_all_handlers(self.root_logger)
 
             # Set up propagation for all existing loggers
@@ -166,8 +189,8 @@ class LogsInterceptor:
                 self._clean_all_handlers(logger)
                 self.patched_loggers.add(logger_name)
 
-        # Set up stdout/stderr redirection
-        self._redirect_stdout_stderr()
+            # Master redirects stdout/stderr
+            self._redirect_stdout_stderr()
 
     def _redirect_stdout_stderr(self) -> None:
         """Redirect stdout and stderr to the logging system."""
@@ -218,15 +241,20 @@ class LogsInterceptor:
         stdout_logger = logging.getLogger("stdout")
         stderr_logger = logging.getLogger("stderr")
 
-        stdout_logger.propagate = False
-        stderr_logger.propagate = False
-
         if self.execution_id:
+            # Child execution: add our handler to stdout/stderr loggers
+            stdout_logger.propagate = False
+            stderr_logger.propagate = False
+
             if self.log_handler not in stdout_logger.handlers:
                 stdout_logger.addHandler(self.log_handler)
             if self.log_handler not in stderr_logger.handlers:
                 stderr_logger.addHandler(self.log_handler)
         else:
+            # Master execution: clean and set up handlers
+            stdout_logger.propagate = False
+            stderr_logger.propagate = False
+
             self._clean_all_handlers(stdout_logger)
             self._clean_all_handlers(stderr_logger)
 
@@ -249,23 +277,22 @@ class LogsInterceptor:
             logging.disable(self.original_disable_level)
 
         # Remove our handler and filter
-        if self.execution_id:
-            if hasattr(self, "execution_filter"):
-                self.log_handler.removeFilter(self.execution_filter)
-            if self.log_handler in self.root_logger.handlers:
-                self.root_logger.removeHandler(self.log_handler)
-
-            # Remove from stdout/stderr loggers too
-            stdout_logger = logging.getLogger("stdout")
-            stderr_logger = logging.getLogger("stderr")
-            if self.log_handler in stdout_logger.handlers:
-                stdout_logger.removeHandler(self.log_handler)
-            if self.log_handler in stderr_logger.handlers:
-                stderr_logger.removeHandler(self.log_handler)
-        else:
-            if self.log_handler in self.root_logger.handlers:
-                self.root_logger.removeHandler(self.log_handler)
+        if self.execution_filter:
+            self.log_handler.removeFilter(self.execution_filter)
+
+        if self.log_handler in self.root_logger.handlers:
+            self.root_logger.removeHandler(self.log_handler)
 
+        # Remove from stdout/stderr loggers
+        stdout_logger = logging.getLogger("stdout")
+        stderr_logger = logging.getLogger("stderr")
+        if self.log_handler in stdout_logger.handlers:
+            stdout_logger.removeHandler(self.log_handler)
+        if self.log_handler in stderr_logger.handlers:
+            stderr_logger.removeHandler(self.log_handler)
+
+        if not self.execution_id:
+            # Master execution: restore everything
             for logger_name in self.patched_loggers:
                 logger = logging.getLogger(logger_name)
                 if self.log_handler in logger.handlers:
@@ -278,6 +305,7 @@ class LogsInterceptor:
 
         self.log_handler.close()
 
+        # Only restore streams if we redirected them
         if self.original_stdout and self.original_stderr:
             sys.stdout = self.original_stdout
             sys.stderr = self.original_stderr

uipath/_services/llm_gateway_service.py

@@ -106,35 +106,32 @@ def _cleanup_schema(model_class: type[BaseModel]) -> Dict[str, Any]:
     """
     schema = model_class.model_json_schema()
 
-    def clean_properties(properties):
-        """Clean property definitions by removing titles and cleaning nested items."""
-        cleaned_props = {}
-        for prop_name, prop_def in properties.items():
-            if isinstance(prop_def, dict):
-                cleaned_prop = {}
-                for key, value in prop_def.items():
-                    if key == "title":  # Skip title
-                        continue
-                    elif key == "items" and isinstance(value, dict):
-                        # Clean nested items
-                        cleaned_items = {}
-                        for item_key, item_value in value.items():
-                            if item_key != "title":
-                                cleaned_items[item_key] = item_value
-                        cleaned_prop[key] = cleaned_items
-                    else:
-                        cleaned_prop[key] = value
-                cleaned_props[prop_name] = cleaned_prop
-        return cleaned_props
+    def clean_type(type_def):
+        """Clean property definitions by removing titles and cleaning nested items. Additionally, `additionalProperties` is ensured on all objects."""
+        cleaned_type = {}
+        for key, value in type_def.items():
+            if key == "title" or key == "properties":
+                continue
+            else:
+                cleaned_type[key] = value
+        if type_def.get("type") == "object" and "additionalProperties" not in type_def:
+            cleaned_type["additionalProperties"] = False
+
+        if "properties" in type_def:
+            properties = type_def.get("properties", {})
+            for key, value in properties.items():
+                properties[key] = clean_type(value)
+            cleaned_type["properties"] = properties
+
+        if "$defs" in type_def:
+            cleaned_defs = {}
+            for key, value in type_def["$defs"].items():
+                cleaned_defs[key] = clean_type(value)
+            cleaned_type["$defs"] = cleaned_defs
+        return cleaned_type
 
     # Create clean schema
-    clean_schema = {
-        "type": "object",
-        "properties": clean_properties(schema.get("properties", {})),
-        "required": schema.get("required", []),
-        "additionalProperties": False,
-    }
-
+    clean_schema = clean_type(schema)
     return clean_schema
 
 

uipath/agent/_utils.py

@@ -4,13 +4,17 @@ from pathlib import PurePath
 from httpx import Response
 from pydantic import TypeAdapter
 
+from uipath._cli._evals._models._evaluation_set import LLMMockingStrategy
 from uipath._cli._utils._studio_project import (
     ProjectFile,
     ProjectFolder,
     StudioClient,
     resolve_path,
 )
-from uipath.agent.models.agent import AgentDefinition
+from uipath.agent.models.agent import (
+    AgentDefinition,
+    UnknownAgentDefinition,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -23,7 +27,7 @@ async def get_file(
     return await studio_client.download_file_async(resolved.id)
 
 
-async def load_agent_definition(project_id: str):
+async def load_agent_definition(project_id: str) -> AgentDefinition:
     studio_client = StudioClient(project_id=project_id)
     project_structure = await studio_client.get_project_structure_async()
 
@@ -100,4 +104,21 @@ async def load_agent_definition(project_id: str):
         "evaluationSets": evaluation_sets,
         **agent,
     }
-    return TypeAdapter(AgentDefinition).validate_python(agent_definition)
+    agent_definition = TypeAdapter(AgentDefinition).validate_python(agent_definition)
+    if agent_definition and isinstance(agent_definition, UnknownAgentDefinition):
+        if agent_definition.evaluation_sets:
+            for evaluation_set in agent_definition.evaluation_sets:
+                for evaluation in evaluation_set.evaluations:
+                    if not evaluation.mocking_strategy:
+                        # Migrate lowCode evaluation definitions
+                        if evaluation.model_extra.get("simulateTools", False):
+                            tools_to_simulate = evaluation.model_extra.get(
+                                "toolsToSimulate", []
+                            )
+                            prompt = evaluation.model_extra.get(
+                                "simulationInstructions", ""
+                            )
+                            evaluation.mocking_strategy = LLMMockingStrategy(
+                                prompt=prompt, tools_to_simulate=tools_to_simulate
+                            )
+    return agent_definition

uipath/agent/models/agent.py

@@ -55,6 +55,7 @@ class AgentToolType(str, Enum):
 
     AGENT = "agent"
     INTEGRATION = "integration"
+    PROCESS = "process"
 
 
 class AgentToolSettings(BaseModel):

{uipath-2.1.72.dist-info → uipath-2.1.74.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: uipath
-Version: 2.1.72
+Version: 2.1.74
 Summary: Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools.
 Project-URL: Homepage, https://uipath.com
 Project-URL: Repository, https://github.com/UiPath/uipath-python
@@ -17,9 +17,12 @@ Requires-Python: >=3.10
 Requires-Dist: azure-monitor-opentelemetry>=1.6.8
 Requires-Dist: click>=8.1.8
 Requires-Dist: httpx>=0.28.1
+Requires-Dist: hydra-core>=1.3.2
+Requires-Dist: mockito>=1.5.4
 Requires-Dist: opentelemetry-instrumentation>=0.52b1
 Requires-Dist: opentelemetry-sdk>=1.31.1
 Requires-Dist: pathlib>=1.0.1
+Requires-Dist: pydantic-function-models>=0.1.10
 Requires-Dist: pydantic>=2.11.1
 Requires-Dist: pyperclip>=1.9.0
 Requires-Dist: python-dotenv>=1.0.1

{uipath-2.1.72.dist-info → uipath-2.1.74.dist-info}/RECORD

@@ -46,18 +46,24 @@ uipath/_cli/_dev/_terminal/_utils/_exporter.py,sha256=oI6D_eMwrh_2aqDYUh4GrJg8VL
 uipath/_cli/_dev/_terminal/_utils/_logger.py,sha256=_ipTl_oAiMF9I7keGt2AAFAMz40DNLVMVkoiq-07UAU,2943
 uipath/_cli/_evals/_evaluator_factory.py,sha256=Gycv94VtGOpMir_Gba-UoiAyrSRfbSfe8_pTfjzcA9Q,3875
 uipath/_cli/_evals/_progress_reporter.py,sha256=hpSt0CXpIoFJGsbqZkqmwyGO_TBNesbWKlvDJUEDxd8,16455
-uipath/_cli/_evals/_runtime.py,sha256=WKcBT6DGzNRjgEOpmH0b7RoEbEsHMyAbcAMs8b_CAI0,11418
-uipath/_cli/_evals/_models/_evaluation_set.py,sha256=RRDaP0X4E8kueL0Io9yB4y8akx3gKZhoSIgTNhgoN9Y,2407
+uipath/_cli/_evals/_runtime.py,sha256=fo-wtUxZ9bvo1ftt_yu7o2oUBvrMqVNKCjaGH556-Vo,11506
+uipath/_cli/_evals/_models/_evaluation_set.py,sha256=XgPNLWciE4FgCYzZXV2kRYHzdtbc33FWSQmZQqVSdMk,4747
 uipath/_cli/_evals/_models/_evaluator.py,sha256=fuC3UOYwPD4d_wdynHeLSCzbu82golNAnnPnxC8Y4rk,3315
 uipath/_cli/_evals/_models/_evaluator_base_params.py,sha256=lTYKOV66tcjW85KHTyOdtF1p1VDaBNemrMAvH8bFIFc,382
 uipath/_cli/_evals/_models/_output.py,sha256=LjwMBGI78sDFa2Dl8b9ReXJmjig57pdLWpuiwChrRLo,3096
 uipath/_cli/_evals/_models/_sw_reporting.py,sha256=tSBLQFAdOIun8eP0vsqt56K6bmCZz_uMaWI3hskg_24,536
 uipath/_cli/_evals/_models/_trajectory_span.py,sha256=8ukM8sB9rvzBMHfC_gnexAC3xlp4uMDevKZrRzcgrm4,3637
+uipath/_cli/_evals/mocks/__init__.py,sha256=2WXwAy_oZw5bKp6L0HB13QygCJeftOB_Bget0AI6Gik,32
+uipath/_cli/_evals/mocks/llm_mocker.py,sha256=EtfPUhKcBRQ7vQPQdF3QanKIx2vzzYWA9PfdjKeJnTw,6108
+uipath/_cli/_evals/mocks/mocker.py,sha256=FjSIqXF5HzQRi1eWrFfXBz-cYZEu5TiMVm2RsKnSEWI,626
+uipath/_cli/_evals/mocks/mocker_factory.py,sha256=V5QKSTtQxztTo4-fK1TyAaXw2Z3mHf2UC5mXqwuUGTs,811
+uipath/_cli/_evals/mocks/mockito_mocker.py,sha256=LtYT6lJM9vc3qtbSZJcUeCzDn4zarkBVj7In_EX7kYY,2087
+uipath/_cli/_evals/mocks/mocks.py,sha256=WqjWtHqKQXAsO1Wwom3Zcr1T09GQygwBWVp-EsxdW8o,4443
 uipath/_cli/_push/sw_file_handler.py,sha256=iE8Sk1Z-9hxmLFFj3j-k4kTK6TzNFP6hUCmxTudG6JQ,18251
 uipath/_cli/_runtime/_contracts.py,sha256=D57cq5V5CZ9p13n_vRDHRcwyJYQUcJLlAMbAOzIiBNI,28932
 uipath/_cli/_runtime/_escalation.py,sha256=x3vI98qsfRA-fL_tNkRVTFXioM5Gv2w0GFcXJJ5eQtg,7981
 uipath/_cli/_runtime/_hitl.py,sha256=VKbM021nVg1HEDnTfucSLJ0LsDn83CKyUtVzofS2qTU,11369
-uipath/_cli/_runtime/_logging.py,sha256=iO0AG_tqUBp7aJZTK_ZgwV3fFvxbi9Rp1UOBn3F76lw,11684
+uipath/_cli/_runtime/_logging.py,sha256=CIJNZH4S8SL980VI3XV9NIFdc97T-KJRT8QE7X1uWYw,12865
 uipath/_cli/_runtime/_runtime.py,sha256=gby9-avNNlEATEfSXtY8FfJ8nREsSCGA4wMgDlSXTDE,2297
 uipath/_cli/_runtime/_script_executor.py,sha256=PjbmEbyCMofGH2F85b8RFsxdV3Tqw0kVqdWOOk2ZLlI,9687
 uipath/_cli/_templates/.psmdcp.template,sha256=C7pBJPt98ovEljcBvGtEUGoWjjQhu9jls1bpYjeLOKA,611
@@ -95,7 +101,7 @@ uipath/_services/documents_service.py,sha256=UnFS8EpOZ_Ng2TZk3OiJJ3iNANvFs7QxuoG
 uipath/_services/entities_service.py,sha256=QKCLE6wRgq3HZraF-M2mljy-8il4vsNHrQhUgkewVVk,14028
 uipath/_services/folder_service.py,sha256=9JqgjKhWD-G_KUnfUTP2BADxL6OK9QNZsBsWZHAULdE,2749
 uipath/_services/jobs_service.py,sha256=tTZNsdZKN3uP7bWPQyBCpJeQxTfuOWbKYOR4L-_yJo4,32736
-uipath/_services/llm_gateway_service.py,sha256=Ka1WCoOBKJfIlm7H9NmbKAPr27UYDYQCTExMbLKMY68,24249
+uipath/_services/llm_gateway_service.py,sha256=IBpkG2N2_9LGHkbajBoet9F26DqILC8zL4uHnnniQvg,24124
 uipath/_services/processes_service.py,sha256=O_uHgQ1rnwiV5quG0OQqabAnE6Rf6cWrMENYY2jKWt8,8585
 uipath/_services/queues_service.py,sha256=VaG3dWL2QK6AJBOLoW2NQTpkPfZjsqsYPl9-kfXPFzA,13534
 uipath/_utils/__init__.py,sha256=VdcpnENJIa0R6Y26NoxY64-wUVyvb4pKfTh1wXDQeMk,526
@@ -109,7 +115,7 @@ uipath/_utils/_ssl_context.py,sha256=xSYitos0eJc9cPHzNtHISX9PBvL6D2vas5G_GiBdLp8
 uipath/_utils/_url.py,sha256=-4eluSrIZCUlnQ3qU17WPJkgaC2KwF9W5NeqGnTNGGo,2512
 uipath/_utils/_user_agent.py,sha256=pVJkFYacGwaQBomfwWVAvBQgdBUo62e4n3-fLIajWUU,563
 uipath/_utils/constants.py,sha256=2xLT-1aW0aJS2USeZbK-7zRgyyi1bgV60L0rtQOUqOM,1721
-uipath/agent/_utils.py,sha256=frpg3LYJofdGWS5w44PsBaNHiW9GRkLokvNnf2D4a54,3309
+uipath/agent/_utils.py,sha256=mf4CtOZch1SFR83Z4QIkjXqMKK8Pm_xV8Q0-RedDkcE,4443
 uipath/agent/conversation/__init__.py,sha256=5hK-Iz131mnd9m6ANnpZZffxXZLVFDQ9GTg5z9ik1oQ,5265
 uipath/agent/conversation/async_stream.py,sha256=BA_8uU1DgE3VpU2KkJj0rkI3bAHLk_ZJKsajR0ipMpo,2055
 uipath/agent/conversation/citation.py,sha256=42dGv-wiYx3Lt7MPuPCFTkjAlSADFSzjyNXuZHdxqvo,2253
@@ -120,7 +126,7 @@ uipath/agent/conversation/exchange.py,sha256=nuk1tEMBHc_skrraT17d8U6AtyJ3h07ExGQ
 uipath/agent/conversation/message.py,sha256=1ZkEs146s79TrOAWCQwzBAEJvjAu4lQBpJ64tKXDgGE,2142
 uipath/agent/conversation/meta.py,sha256=3t0eS9UHoAPHre97QTUeVbjDhnMX4zj4-qG6ju0B8wY,315
 uipath/agent/conversation/tool.py,sha256=ol8XI8AVd-QNn5auXNBPcCzOkh9PPFtL7hTK3kqInkU,2191
-uipath/agent/models/agent.py,sha256=ie_N47K-txv9Q9kXeynI1FsO2l120JYmjPXNm-e6yMo,12952
+uipath/agent/models/agent.py,sha256=a4tE5LTne0ghpb9qeKkvBAotFJSC88hQP-y7OKGrYXk,12976
 uipath/eval/_helpers/__init__.py,sha256=GSmZMryjuO3Wo_zdxZdrHCRRsgOxsVFYkYgJ15YNC3E,86
 uipath/eval/_helpers/helpers.py,sha256=iE2HHdMiAdAMLqxHkPKHpfecEtAuN5BTBqvKFTI8ciE,1315
 uipath/eval/evaluators/__init__.py,sha256=DJAAhgv0I5UfBod4sGnSiKerfrz1iMmk7GNFb71V8eI,494
@@ -159,8 +165,8 @@ uipath/tracing/_traced.py,sha256=yBIY05PCCrYyx50EIHZnwJaKNdHPNx-YTR1sHQl0a98,199
 uipath/tracing/_utils.py,sha256=qd7N56tg6VXQ9pREh61esBgUWLNA0ssKsE0QlwrRWFM,11974
 uipath/utils/__init__.py,sha256=VD-KXFpF_oWexFg6zyiWMkxl2HM4hYJMIUDZ1UEtGx0,105
 uipath/utils/_endpoints_manager.py,sha256=iRTl5Q0XAm_YgcnMcJOXtj-8052sr6jpWuPNz6CgT0Q,8408
-uipath-2.1.72.dist-info/METADATA,sha256=sIUVm86o9y3uorIq58WEe4g-58aQ_tD0mp_8TPcrhHc,6482
-uipath-2.1.72.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-uipath-2.1.72.dist-info/entry_points.txt,sha256=9C2_29U6Oq1ExFu7usihR-dnfIVNSKc-0EFbh0rskB4,43
-uipath-2.1.72.dist-info/licenses/LICENSE,sha256=-KBavWXepyDjimmzH5fVAsi-6jNVpIKFc2kZs0Ri4ng,1058
-uipath-2.1.72.dist-info/RECORD,,
+uipath-2.1.74.dist-info/METADATA,sha256=YI-VaC3UPoQGy3mDnKYd4in3JnlExWwD9QzMJdO-H6s,6593
+uipath-2.1.74.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+uipath-2.1.74.dist-info/entry_points.txt,sha256=9C2_29U6Oq1ExFu7usihR-dnfIVNSKc-0EFbh0rskB4,43
+uipath-2.1.74.dist-info/licenses/LICENSE,sha256=-KBavWXepyDjimmzH5fVAsi-6jNVpIKFc2kZs0Ri4ng,1058
+uipath-2.1.74.dist-info/RECORD,,