uipath 2.1.71__py3-none-any.whl → 2.1.73__py3-none-any.whl

uipath/_cli/_evals/mocks/mocker_factory.py

@@ -0,0 +1,25 @@
+"""Mocker Factory."""
+
+from uipath._cli._evals._models._evaluation_set import (
+    EvaluationItem,
+    LLMMockingStrategy,
+    MockitoMockingStrategy,
+)
+from uipath._cli._evals.mocks.llm_mocker import LLMMocker
+from uipath._cli._evals.mocks.mocker import Mocker
+from uipath._cli._evals.mocks.mockito_mocker import MockitoMocker
+
+
+class MockerFactory:
+    """Mocker factory."""
+
+    @staticmethod
+    def create(evaluation_item: EvaluationItem) -> Mocker:
+        """Create a mocker instance."""
+        match evaluation_item.mocking_strategy:
+            case LLMMockingStrategy():
+                return LLMMocker(evaluation_item)
+            case MockitoMockingStrategy():
+                return MockitoMocker(evaluation_item)
+            case _:
+                raise ValueError("Unknown mocking strategy")

uipath/_cli/_evals/mocks/mockito_mocker.py

@@ -0,0 +1,62 @@
+"""Mockito mocker implementation.
+
+https://mockito-python.readthedocs.io/en/latest/
+"""
+
+from typing import Any, Callable
+
+from hydra.utils import instantiate
+from mockito import invocation, mocking  # type: ignore[import-untyped]
+
+from uipath._cli._evals._models._evaluation_set import (
+    EvaluationItem,
+    MockingAnswerType,
+    MockitoMockingStrategy,
+)
+from uipath._cli._evals.mocks.mocker import Mocker, R, T
+
+
+class Stub:
+    """Stub interface."""
+
+    def __getattr__(self, item):
+        """Return a wrapper function that raises an exception."""
+
+        def func(*_args, **_kwargs):
+            """Not Implemented."""
+            raise NotImplementedError()
+
+        return func
+
+
+class MockitoMocker(Mocker):
+    """Mockito Mocker."""
+
+    def __init__(self, evaluation_item: EvaluationItem):
+        """Instantiate a mockito mocker."""
+        self.evaluation_item = evaluation_item
+        assert isinstance(self.evaluation_item.mocking_strategy, MockitoMockingStrategy)
+
+        self.stub = Stub()
+        mock_obj = mocking.Mock(self.stub)
+
+        for behavior in self.evaluation_item.mocking_strategy.behaviors:
+            stubbed = invocation.StubbedInvocation(mock_obj, behavior.function)(
+                *instantiate(behavior.arguments.args, _convert_="object"),
+                **instantiate(behavior.arguments.kwargs, _convert_="object"),
+            )
+            for answer in behavior.then:
+                if answer.type == MockingAnswerType.RETURN:
+                    stubbed = stubbed.thenReturn(
+                        instantiate(answer.model_dump(), _convert_="object")["value"]
+                    )
+                elif answer.type == MockingAnswerType.RAISE:
+                    stubbed = stubbed.thenRaise(
+                        instantiate(answer.model_dump(), _convert_="object")["value"]
+                    )
+
+    async def response(
+        self, func: Callable[[T], R], params: dict[str, Any], *args: T, **kwargs
+    ) -> R:
+        """Respond with mocked response."""
+        return getattr(self.stub, params["name"])(*args, **kwargs)

uipath/_cli/_evals/mocks/mocks.py

@@ -0,0 +1,136 @@
+"""Mocking interface."""
+
+import asyncio
+import functools
+import inspect
+import logging
+import threading
+from contextvars import ContextVar
+from typing import Any, Callable, Optional
+
+from pydantic import TypeAdapter
+from pydantic_function_models import ValidatedFunction  # type: ignore[import-untyped]
+
+from uipath._cli._evals._models._evaluation_set import EvaluationItem
+from uipath._cli._evals.mocks.mocker import Mocker
+from uipath._cli._evals.mocks.mocker_factory import MockerFactory
+
+evaluation_context: ContextVar[Optional[EvaluationItem]] = ContextVar(
+    "evaluation", default=None
+)
+
+mocker_context: ContextVar[Optional[Mocker]] = ContextVar("mocker", default=None)
+
+
+def set_evaluation_item(item: EvaluationItem) -> None:
+    """Set an evaluation item within an evaluation set."""
+    evaluation_context.set(item)
+    try:
+        mocker_context.set(MockerFactory.create(item))
+    except Exception:
+        logger.warning(f"Failed to create mocker for evaluation {item.name}")
+        mocker_context.set(None)
+
+
+async def get_mocked_response(
+    func: Callable[[Any], Any], params: dict[str, Any], *args, **kwargs
+) -> Any:
+    """Get a mocked response."""
+    mocker = mocker_context.get()
+    evaluation_item = evaluation_context.get()
+    if mocker is None or evaluation_item is None:
+        # TODO raise a new UiPath exception type
+        raise RuntimeError(f"Evaluation item {func.__name__} has not been evaluated")
+    else:
+        return await mocker.response(func, params, *args, **kwargs)
+
+
+_event_loop = None
+logger = logging.getLogger(__name__)
+
+
+def run_coroutine(coro):
+    """Run a coroutine synchronously."""
+    global _event_loop
+    if not _event_loop or not _event_loop.is_running():
+        _event_loop = asyncio.new_event_loop()
+        threading.Thread(target=_event_loop.run_forever, daemon=True).start()
+    future = asyncio.run_coroutine_threadsafe(coro, _event_loop)
+    return future.result()
+
+
+def mocked_response_decorator(func, params: dict[str, Any]):
+    """Mocked response decorator."""
+
+    async def mock_response_generator(*args, **kwargs):
+        mocked_response = await get_mocked_response(func, params, *args, **kwargs)
+        return_type: Any = func.__annotations__.get("return", None)
+
+        if return_type is not None:
+            mocked_response = TypeAdapter(return_type).validate_python(mocked_response)
+        return mocked_response
+
+    is_async = inspect.iscoroutinefunction(func)
+    if is_async:
+
+        @functools.wraps(func)
+        async def decorated_func(*args, **kwargs):
+            try:
+                return await mock_response_generator(*args, **kwargs)
+            except Exception:
+                logger.warning(
+                    f"Failed to mock response for {func.__name__}. Falling back to func."
+                )
+                return await func(*args, **kwargs)
+    else:
+
+        @functools.wraps(func)
+        def decorated_func(*args, **kwargs):
+            try:
+                return run_coroutine(mock_response_generator(*args, **kwargs))
+            except Exception:
+                logger.warning(
+                    f"Failed to mock response for {func.__name__}. Falling back to func."
+                )
+                return func(*args, **kwargs)
+
+    return decorated_func
+
+
+def mockable(
+    name: Optional[str] = None,
+    description: Optional[str] = None,
+    **kwargs,
+):
+    """Decorate a function to be a mockable."""
+
+    def decorator(func):
+        params = {
+            "name": name or func.__name__,
+            "description": description or func.__doc__,
+            "input_schema": get_input_schema(func),
+            "output_schema": get_output_schema(func),
+            **kwargs,
+        }
+        return mocked_response_decorator(func, params)
+
+    return decorator
+
+
+def get_output_schema(func):
+    """Retrieves the JSON schema for a function's return type hint."""
+    try:
+        adapter = TypeAdapter(inspect.signature(func).return_annotation)
+        return adapter.json_schema()
+    except Exception:
+        logger.warning(f"Unable to extract output schema for function {func.__name__}")
+        return {}
+
+
+def get_input_schema(func):
+    """Retrieves the JSON schema for a function's input type."""
+    try:
+        return ValidatedFunction(func).model.model_json_schema()
+    except Exception:
+        logger.warning(f"Unable to extract input schema for function {func.__name__}")
+        return {}

uipath/_cli/_runtime/_contracts.py

@@ -525,6 +525,7 @@ class UiPathBaseRuntime(ABC):
             dir=self.context.runtime_dir,
             file=self.context.logs_file,
             job_id=self.context.job_id,
+            execution_id=self.context.execution_id,
             is_debug_run=self.is_debug_run(),
             log_handler=self.context.log_handler,
         )

uipath/_cli/_runtime/_logging.py

@@ -1,8 +1,14 @@
 import logging
 import os
 import sys
+from contextvars import ContextVar
 from typing import Optional, TextIO, Union, cast
 
+# Context variable to track current execution_id
+current_execution_id: ContextVar[Optional[str]] = ContextVar(
+    "current_execution_id", default=None
+)
+
 
 class PersistentLogsHandler(logging.FileHandler):
     """A simple log handler that always writes to a single file without rotation."""
@@ -20,6 +26,30 @@ class PersistentLogsHandler(logging.FileHandler):
         self.setFormatter(self.formatter)
 
 
+class ExecutionContextFilter(logging.Filter):
+    """Filter that only allows logs from a specific execution context."""
+
+    def __init__(self, execution_id: str):
+        super().__init__()
+        self.execution_id = execution_id
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        """Allow logs that have matching execution_id attribute or context."""
+        # First check if record has execution_id attribute
+        record_execution_id = getattr(record, "execution_id", None)
+        if record_execution_id == self.execution_id:
+            return True
+
+        # Fall back to context variable
+        ctx_execution_id = current_execution_id.get()
+        if ctx_execution_id == self.execution_id:
+            # Inject execution_id into record for downstream handlers
+            record.execution_id = self.execution_id
+            return True
+
+        return False
+
+
 class LogsInterceptor:
     """Intercepts all logging and stdout/stderr, routing to either persistent log files or stdout based on whether it's running as a job or not."""
 
@@ -31,6 +61,7 @@ class LogsInterceptor:
         job_id: Optional[str] = None,
         is_debug_run: bool = False,
         log_handler: Optional[logging.Handler] = None,
+        execution_id: Optional[str] = None,
     ):
         """Initialize the log interceptor.
 
@@ -41,9 +72,11 @@ class LogsInterceptor:
             job_id (str, optional): If provided, logs go to file; otherwise, to stdout.
             is_debug_run (bool, optional): If True, log the output to stdout/stderr.
             log_handler (logging.Handler, optional): Custom log handler to use.
+            execution_id (str, optional): Unique identifier for this execution context.
         """
         min_level = min_level or "INFO"
         self.job_id = job_id
+        self.execution_id = execution_id
 
         # Convert to numeric level for consistent comparison
         self.numeric_min_level = getattr(logging, min_level.upper(), logging.INFO)
@@ -81,6 +114,12 @@ class LogsInterceptor:
                 self.log_handler = PersistentLogsHandler(file=log_file)
 
         self.log_handler.setLevel(self.numeric_min_level)
+
+        # Add execution context filter if execution_id provided
+        if execution_id:
+            self.execution_filter = ExecutionContextFilter(execution_id)
+            self.log_handler.addFilter(self.execution_filter)
+
         self.logger = logging.getLogger("runtime")
         self.patched_loggers: set[str] = set()
 
@@ -95,22 +134,37 @@ class LogsInterceptor:
 
     def setup(self) -> None:
         """Configure logging to use our persistent handler."""
-        # Use global disable to prevent all logging below our minimum level
-        if self.numeric_min_level > logging.NOTSET:
+        # Set the context variable for this execution
+        if self.execution_id:
+            current_execution_id.set(self.execution_id)
+
+        # Only use global disable if we're not in a parallel execution context
+        if not self.execution_id and self.numeric_min_level > logging.NOTSET:
             logging.disable(self.numeric_min_level - 1)
 
         # Set root logger level
         self.root_logger.setLevel(self.numeric_min_level)
 
-        # Remove ALL handlers from root logger and add only ours
-        self._clean_all_handlers(self.root_logger)
+        if self.execution_id:
+            # Parallel execution mode: add our handler without removing others
+            if self.log_handler not in self.root_logger.handlers:
+                self.root_logger.addHandler(self.log_handler)
+
+            # Set up propagation for all existing loggers
+            for logger_name in logging.root.manager.loggerDict:
+                logger = logging.getLogger(logger_name)
+                # Keep propagation enabled so logs flow to all handlers
+                self.patched_loggers.add(logger_name)
+        else:
+            # Single execution mode: remove all handlers and add only ours
+            self._clean_all_handlers(self.root_logger)
 
-        # Set up propagation for all existing loggers
-        for logger_name in logging.root.manager.loggerDict:
-            logger = logging.getLogger(logger_name)
-            logger.propagate = False  # Prevent double-logging
-            self._clean_all_handlers(logger)
-            self.patched_loggers.add(logger_name)
+            # Set up propagation for all existing loggers
+            for logger_name in logging.root.manager.loggerDict:
+                logger = logging.getLogger(logger_name)
+                logger.propagate = False  # Prevent double-logging
+                self._clean_all_handlers(logger)
+                self.patched_loggers.add(logger_name)
 
         # Set up stdout/stderr redirection
         self._redirect_stdout_stderr()
@@ -130,7 +184,7 @@ class LogsInterceptor:
                 self.level = level
                 self.min_level = min_level
                 self.buffer = ""
-                self.sys_file = sys_file  # Store reference to system stdout/stderr
+                self.sys_file = sys_file
 
             def write(self, message: str) -> None:
                 self.buffer += message
@@ -138,7 +192,7 @@ class LogsInterceptor:
                     line, self.buffer = self.buffer.split("\n", 1)
                     # Only log if the message is not empty and the level is sufficient
                     if line and self.level >= self.min_level:
-                        # Use _log to avoid potential recursive logging if logging methods are overridden
+                        # The context variable is automatically available here
                         self.logger._log(self.level, line, ())
 
             def flush(self) -> None:
@@ -160,14 +214,21 @@ class LogsInterceptor:
             def writable(self) -> bool:
                 return True
 
-        # Set up stdout and stderr loggers with propagate=False
+        # Set up stdout and stderr loggers
         stdout_logger = logging.getLogger("stdout")
-        stdout_logger.propagate = False
-        self._clean_all_handlers(stdout_logger)
-
         stderr_logger = logging.getLogger("stderr")
+
+        stdout_logger.propagate = False
         stderr_logger.propagate = False
-        self._clean_all_handlers(stderr_logger)
+
+        if self.execution_id:
+            if self.log_handler not in stdout_logger.handlers:
+                stdout_logger.addHandler(self.log_handler)
+            if self.log_handler not in stderr_logger.handlers:
+                stderr_logger.addHandler(self.log_handler)
+        else:
+            self._clean_all_handlers(stdout_logger)
+            self._clean_all_handlers(stderr_logger)
 
         # Use the min_level in the LoggerWriter to filter messages
         sys.stdout = LoggerWriter(
@@ -179,21 +240,41 @@ class LogsInterceptor:
 
     def teardown(self) -> None:
         """Restore original logging configuration."""
-        # Restore the original disable level
-        logging.disable(self.original_disable_level)
-
-        if self.log_handler in self.root_logger.handlers:
-            self.root_logger.removeHandler(self.log_handler)
+        # Clear the context variable
+        if self.execution_id:
+            current_execution_id.set(None)
 
-        for logger_name in self.patched_loggers:
-            logger = logging.getLogger(logger_name)
-            if self.log_handler in logger.handlers:
-                logger.removeHandler(self.log_handler)
-
-        self.root_logger.setLevel(self.original_level)
-        for handler in self.original_handlers:
-            if handler not in self.root_logger.handlers:
-                self.root_logger.addHandler(handler)
+        # Restore the original disable level
+        if not self.execution_id:
+            logging.disable(self.original_disable_level)
+
+        # Remove our handler and filter
+        if self.execution_id:
+            if hasattr(self, "execution_filter"):
+                self.log_handler.removeFilter(self.execution_filter)
+            if self.log_handler in self.root_logger.handlers:
+                self.root_logger.removeHandler(self.log_handler)
+
+            # Remove from stdout/stderr loggers too
+            stdout_logger = logging.getLogger("stdout")
+            stderr_logger = logging.getLogger("stderr")
+            if self.log_handler in stdout_logger.handlers:
+                stdout_logger.removeHandler(self.log_handler)
+            if self.log_handler in stderr_logger.handlers:
+                stderr_logger.removeHandler(self.log_handler)
+        else:
+            if self.log_handler in self.root_logger.handlers:
+                self.root_logger.removeHandler(self.log_handler)
+
+            for logger_name in self.patched_loggers:
+                logger = logging.getLogger(logger_name)
+                if self.log_handler in logger.handlers:
+                    logger.removeHandler(self.log_handler)
+
+            self.root_logger.setLevel(self.original_level)
+            for handler in self.original_handlers:
+                if handler not in self.root_logger.handlers:
+                    self.root_logger.addHandler(handler)
 
         self.log_handler.close()
 

uipath/_cli/cli_pull.py

@@ -112,7 +112,7 @@ async def download_folder_files(
             if local_hash != remote_hash:
                 styled_path = click.style(str(file_path), fg="cyan")
                 console.warning(f"File {styled_path}" + " differs from remote version.")
-                response = click.prompt("Do you want to override it? (y/n)", type=str)
+                response = click.prompt("Do you want to overwrite it? (y/n)", type=str)
                 if response.lower() == "y":
                     with open(local_path, "w", encoding="utf-8", newline="\n") as f:
                         f.write(remote_content)

uipath/_services/llm_gateway_service.py

@@ -106,35 +106,32 @@ def _cleanup_schema(model_class: type[BaseModel]) -> Dict[str, Any]:
     """
     schema = model_class.model_json_schema()
 
-    def clean_properties(properties):
-        """Clean property definitions by removing titles and cleaning nested items."""
-        cleaned_props = {}
-        for prop_name, prop_def in properties.items():
-            if isinstance(prop_def, dict):
-                cleaned_prop = {}
-                for key, value in prop_def.items():
-                    if key == "title":  # Skip title
-                        continue
-                    elif key == "items" and isinstance(value, dict):
-                        # Clean nested items
-                        cleaned_items = {}
-                        for item_key, item_value in value.items():
-                            if item_key != "title":
-                                cleaned_items[item_key] = item_value
-                        cleaned_prop[key] = cleaned_items
-                    else:
-                        cleaned_prop[key] = value
-                cleaned_props[prop_name] = cleaned_prop
-        return cleaned_props
+    def clean_type(type_def):
+        """Clean property definitions by removing titles and cleaning nested items. Additionally, `additionalProperties` is ensured on all objects."""
+        cleaned_type = {}
+        for key, value in type_def.items():
+            if key == "title" or key == "properties":
+                continue
+            else:
+                cleaned_type[key] = value
+        if type_def.get("type") == "object" and "additionalProperties" not in type_def:
+            cleaned_type["additionalProperties"] = False
+
+        if "properties" in type_def:
+            properties = type_def.get("properties", {})
+            for key, value in properties.items():
+                properties[key] = clean_type(value)
+            cleaned_type["properties"] = properties
+
+        if "$defs" in type_def:
+            cleaned_defs = {}
+            for key, value in type_def["$defs"].items():
+                cleaned_defs[key] = clean_type(value)
+            cleaned_type["$defs"] = cleaned_defs
+        return cleaned_type
 
     # Create clean schema
-    clean_schema = {
-        "type": "object",
-        "properties": clean_properties(schema.get("properties", {})),
-        "required": schema.get("required", []),
-        "additionalProperties": False,
-    }
-
+    clean_schema = clean_type(schema)
     return clean_schema
 
 

uipath/agent/_utils.py

@@ -4,13 +4,17 @@ from pathlib import PurePath
 from httpx import Response
 from pydantic import TypeAdapter
 
+from uipath._cli._evals._models._evaluation_set import LLMMockingStrategy
 from uipath._cli._utils._studio_project import (
     ProjectFile,
     ProjectFolder,
     StudioClient,
     resolve_path,
 )
-from uipath.agent.models.agent import AgentDefinition
+from uipath.agent.models.agent import (
+    AgentDefinition,
+    UnknownAgentDefinition,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -23,7 +27,7 @@ async def get_file(
     return await studio_client.download_file_async(resolved.id)
 
 
-async def load_agent_definition(project_id: str):
+async def load_agent_definition(project_id: str) -> AgentDefinition:
     studio_client = StudioClient(project_id=project_id)
     project_structure = await studio_client.get_project_structure_async()
 
@@ -31,6 +35,52 @@ async def load_agent_definition(project_id: str):
         await get_file(project_structure, PurePath("agent.json"), studio_client)
     ).json()
 
+    evaluators = []
+    try:
+        evaluators_path = resolve_path(
+            project_structure, PurePath("evals", "evaluators")
+        )
+        if isinstance(evaluators_path, ProjectFolder):
+            for file in evaluators_path.files:
+                evaluators.append(
+                    (
+                        await get_file(
+                            evaluators_path, PurePath(file.name), studio_client
+                        )
+                    ).json()
+                )
+        else:
+            logger.warning(
+                "Unable to read evaluators from project. Defaulting to empty evaluators."
+            )
+    except Exception:
+        logger.warning(
+            "Unable to read evaluators from project. Defaulting to empty evaluators."
+        )
+
+    evaluation_sets = []
+    try:
+        evaluation_sets_path = resolve_path(
+            project_structure, PurePath("evals", "eval-sets")
+        )
+        if isinstance(evaluation_sets_path, ProjectFolder):
+            for file in evaluation_sets_path.files:
+                evaluation_sets.append(
+                    (
+                        await get_file(
+                            evaluation_sets_path, PurePath(file.name), studio_client
+                        )
+                    ).json()
+                )
+        else:
+            logger.warning(
+                "Unable to read eval-sets from project. Defaulting to empty eval-sets."
+            )
+    except Exception:
+        logger.warning(
+            "Unable to read eval-sets from project. Defaulting to empty eval-sets."
+        )
+
     resolved_path = resolve_path(project_structure, PurePath("resources"))
     if isinstance(resolved_path, ProjectFolder):
         resource_folders = resolved_path.folders
@@ -50,6 +100,25 @@ async def load_agent_definition(project_id: str):
         "id": project_id,
         "name": project_structure.name,
         "resources": resources,
+        "evaluators": evaluators,
+        "evaluationSets": evaluation_sets,
         **agent,
     }
-    return TypeAdapter(AgentDefinition).validate_python(agent_definition)
+    agent_definition = TypeAdapter(AgentDefinition).validate_python(agent_definition)
+    if agent_definition and isinstance(agent_definition, UnknownAgentDefinition):
+        if agent_definition.evaluation_sets:
+            for evaluation_set in agent_definition.evaluation_sets:
+                for evaluation in evaluation_set.evaluations:
+                    if not evaluation.mocking_strategy:
+                        # Migrate lowCode evaluation definitions
+                        if evaluation.model_extra.get("simulateTools", False):
+                            tools_to_simulate = evaluation.model_extra.get(
+                                "toolsToSimulate", []
+                            )
+                            prompt = evaluation.model_extra.get(
+                                "simulationInstructions", ""
+                            )
+                            evaluation.mocking_strategy = LLMMockingStrategy(
+                                prompt=prompt, tools_to_simulate=tools_to_simulate
+                            )
+    return agent_definition

uipath/agent/models/agent.py

@@ -5,6 +5,8 @@ from typing import Annotated, Any, Dict, List, Literal, Optional, Union
 
 from pydantic import BaseModel, ConfigDict, Discriminator, Field, Tag
 
+from uipath._cli._evals._models._evaluation_set import EvaluationSet
+from uipath._cli._evals._models._evaluator import Evaluator
 from uipath.models import Connection
 
 
@@ -53,6 +55,7 @@ class AgentToolType(str, Enum):
 
     AGENT = "agent"
     INTEGRATION = "integration"
+    PROCESS = "process"
 
 
 class AgentToolSettings(BaseModel):
@@ -307,6 +310,14 @@ class BaseAgentDefinition(BaseModel):
     resources: List[AgentResourceConfig] = Field(
         ..., description="List of tools, context, and escalation resources"
     )
+    evaluation_sets: Optional[List[EvaluationSet]] = Field(
+        None,
+        alias="evaluationSets",
+        description="List of agent evaluation sets",
+    )
+    evaluators: Optional[List[Evaluator]] = Field(
+        None, description="List of agent evaluators"
+    )
 
     model_config = ConfigDict(
         validate_by_name=True, validate_by_alias=True, extra="allow"